From d7103d2e6533a953a2d7ac661c0f37382a1460af Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:03:22 +0800 Subject: [PATCH 01/10] Add deterministic H3 FFN down projection with H100 evidence Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- .gitignore | 1 + .../h100-cancellation.json | 1516 ++++++++++++ .../h3-down-h100-20261010/h100-random.json | 1516 ++++++++++++ .../h3-down-h100-20261010/h100-replays.json | 2118 +++++++++++++++++ .../h3-down-h100-20261010/h100-summary.json | 672 ++++++ .../h100-swiglu-like.json | 1516 ++++++++++++ docs/operators/h3-ffn-down.md | 163 ++ rl_engine/kernels/ops/h3_down_fp32_backend.py | 70 + rl_engine/kernels/ops/h3_down_validation.py | 120 + rl_engine/kernels/ops/h3_ffn_down.py | 256 ++ rl_engine/kernels/registry.py | 21 + scripts/plot_h3_down_report.py | 143 ++ scripts/prepare_h3_down_fixture.py | 111 + scripts/requirements-h3-down.txt | 7 + scripts/run_h3_down_h100.sh | 78 + scripts/validate_h3_ffn_down.py | 451 ++++ tests/test_h3_down_fixture.py | 70 + tests/test_h3_down_tooling.py | 116 + tests/test_h3_ffn_down.py | 277 +++ 19 files changed, 9222 insertions(+) create mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json create mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-random.json create mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-replays.json create mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-summary.json create mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json create mode 100644 docs/operators/h3-ffn-down.md create mode 100644 rl_engine/kernels/ops/h3_down_fp32_backend.py create mode 100644 rl_engine/kernels/ops/h3_down_validation.py create mode 100644 rl_engine/kernels/ops/h3_ffn_down.py create mode 100644 scripts/plot_h3_down_report.py create mode 100644 scripts/prepare_h3_down_fixture.py create mode 100644 scripts/requirements-h3-down.txt create mode 100644 scripts/run_h3_down_h100.sh create mode 100644 scripts/validate_h3_ffn_down.py create mode 100644 tests/test_h3_down_fixture.py create mode 100644 tests/test_h3_down_tooling.py create mode 100644 tests/test_h3_ffn_down.py diff --git a/.gitignore b/.gitignore index 1a96f7df2..c24a4acb2 100644 --- a/.gitignore +++ b/.gitignore @@ -210,6 +210,7 @@ __marimo__/ # Cross-configuration alignment local run artifacts /runs/ +/artifacts/h3-down/ # Local dev notes (not for upstream) _dev_notes/ diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json b/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json new file mode 100644 index 000000000..4dbb13f00 --- /dev/null +++ b/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json @@ -0,0 +1,1516 @@ +{ + "schema": "h3-down-measured-case-part-v1", + "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", + "case_indexes": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47], + "cases": [ + { + "rows": 1, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.52587890625e-05,"max_error_over_tolerance": 0.00030462356517091393,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0004891455173492432,"max_error_over_tolerance": 0.0047654458321630955,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.03244646266102791,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5236491560935974, + "samples_ms": [0.5236491560935974, 0.5030222237110138, 0.5003213882446289, 0.5344152450561523, 0.7693693041801453], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.21550804376602173, + "samples_ms": [0.5729421973228455, 0.3075897693634033, 0.21550804376602173, 0.21023675799369812, 0.21257996559143066], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.29595568776130676, + "samples_ms": [0.4697144031524658, 0.29595568776130676, 0.2908147871494293, 0.3237724304199219, 0.23474544286727905], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.9405077695846558, + "samples_ms": [4.387270659208298, 4.102259874343872, 3.798823803663254, 3.9236098527908325, 3.9405077695846558], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08775666356086731, + "samples_ms": [0.10928884148597717, 0.09210780262947083, 0.08775666356086731, 0.08741766214370728, 0.0855475664138794], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.7773749530315399, + "samples_ms": [1.0754838585853577, 0.7773749530315399, 0.8160248398780823, 0.6982982158660889, 0.47647953033447266], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 31, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5269499272108078e-05,"max_error_over_tolerance": 0.00030487452750094235,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009725391864776611,"max_error_over_tolerance": 0.009215985424816608,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007811069488525391,"max_error_over_tolerance": 0.05571417137980461,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5122534930706024, + "samples_ms": [0.553864985704422, 0.5489625036716461, 0.5122534930706024, 0.4660859704017639, 0.45041367411613464], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19108504056930542, + "samples_ms": [0.19708648324012756, 0.19152089953422546, 0.19108504056930542, 0.18773600459098816, 0.18952414393424988], + "peak_allocated_bytes": 224250880, + "incremental_peak_bytes": 888832, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.16224011778831482, + "samples_ms": [0.17355382442474365, 0.16224011778831482, 0.15947222709655762, 0.15872344374656677, 0.16421452164649963], + "peak_allocated_bytes": 378764288, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.283212125301361, + "samples_ms": [4.283212125301361, 4.39327210187912, 4.2417012155056, 4.258990287780762, 4.3707191944122314], + "peak_allocated_bytes": 537462784, + "incremental_peak_bytes": 314100736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09158998727798462, + "samples_ms": [0.10312721133232117, 0.09158998727798462, 0.08689984679222107, 0.08834898471832275, 0.0958070158958435], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2784041464328766, + "samples_ms": [1.018371433019638, 1.3990290462970734, 1.0924860835075378, 1.2784041464328766, 1.558348536491394], + "peak_allocated_bytes": 378724864, + "incremental_peak_bytes": 155362816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 31,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "dceb60bc57d87aedd2f14d50ca48ec4f6bf5ebd8620a62556cc66e83edf007bc", + "dtype": "torch.bfloat16", + "shape": [31, 5376] + }, + "dx": { + "sha256": "1b6fbf79020c8c439afedaf519fbbc705df1f51d1cdd7bdb9350d3bae86e7e9a", + "dtype": "torch.bfloat16", + "shape": [31, 14336] + }, + "dw": { + "sha256": "cdb1d812e7bed96785862d814fae7f6331a1dfcbb93b80d71ee4d48910dff433", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5269499272108078e-05,"max_error_over_tolerance": 0.00030487452750094235,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009725391864776611,"max_error_over_tolerance": 0.009215985424816608,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.05573455989360809,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5172304809093475, + "samples_ms": [0.5358867347240448, 0.5300790071487427, 0.5172304809093475, 0.49916282296180725, 0.5000084638595581], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22250041365623474, + "samples_ms": [0.24210289120674133, 0.22250041365623474, 0.22366270422935486, 0.2074204385280609, 0.20667538046836853], + "peak_allocated_bytes": 224347648, + "incremental_peak_bytes": 917504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.15189871191978455, + "samples_ms": [0.16543269157409668, 0.15189871191978455, 0.14640763401985168, 0.1889653503894806, 0.1428835093975067], + "peak_allocated_bytes": 377570816, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.7806369364261627, + "samples_ms": [3.911960870027542, 3.7806369364261627, 3.7540942430496216, 3.8062632083892822, 3.6593489348888397], + "peak_allocated_bytes": 536416256, + "incremental_peak_bytes": 312986112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08858740329742432, + "samples_ms": [0.10128691792488098, 0.09026005864143372, 0.08858740329742432, 0.08736178278923035, 0.0874735414981842], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8712038397789001, + "samples_ms": [0.9313859045505524, 0.830329954624176, 0.9034276008605957, 0.8625313639640808, 0.8712038397789001], + "peak_allocated_bytes": 378832384, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 32,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "97a7cecd5fcabd2832fe2a7f66e17a7a18bfa2ea038e74ba17d39a8b5faf750d", + "dtype": "torch.bfloat16", + "shape": [32, 5376] + }, + "dx": { + "sha256": "839af44e35ad707cd7d78d8d1c84034cc6caaf890d04abd959025a9e2c9197ea", + "dtype": "torch.bfloat16", + "shape": [32, 14336] + }, + "dw": { + "sha256": "4285d8027692d41f18b2c737bf635a5371d268a17b44ea6e93f5cb196ac945cd", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.527508720755577e-05,"max_error_over_tolerance": 0.00030498611158691347,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000972449779510498,"max_error_over_tolerance": 0.009215136989951134,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.055741358548402786,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.45315176248550415, + "samples_ms": [0.4736557602882385, 0.45315176248550415, 0.45205652713775635, 0.4499666392803192, 0.4560798406600952], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.18961727619171143, + "samples_ms": [0.20431727170944214, 0.19268319010734558, 0.18948689103126526, 0.18567591905593872, 0.18961727619171143], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.1841410994529724, + "samples_ms": [0.2075396478176117, 0.19084662199020386, 0.1840628683567047, 0.1841410994529724, 0.17883256077766418], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.174433648586273, + "samples_ms": [4.592347890138626, 4.004254937171936, 4.059191793203354, 4.174433648586273, 4.196617752313614], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08811056613922119, + "samples_ms": [0.10736286640167236, 0.09493529796600342, 0.08811056613922119, 0.08622929453849792, 0.08698925375938416], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8363053202629089, + "samples_ms": [0.872097909450531, 0.9182840585708618, 0.81629678606987, 0.773504376411438, 0.8363053202629089], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 127, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5271827578544617e-05,"max_error_over_tolerance": 0.0003049470251426101,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000971376895904541,"max_error_over_tolerance": 0.009242621250450611,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.08660231530666351,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.517033040523529, + "samples_ms": [0.517033040523529, 0.569421797990799, 0.5456507205963135, 0.4870966076850891, 0.45920535922050476], + "peak_allocated_bytes": 231267328, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19916146993637085, + "samples_ms": [0.2094767987728119, 0.20661205053329468, 0.19916146993637085, 0.19781291484832764, 0.1978762447834015], + "peak_allocated_bytes": 233998336, + "incremental_peak_bytes": 4096512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2178773283958435, + "samples_ms": [0.2397596836090088, 0.23138150572776794, 0.2178773283958435, 0.21693482995033264, 0.21483004093170166], + "peak_allocated_bytes": 389515264, + "incremental_peak_bytes": 159613440, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.137318581342697, + "samples_ms": [3.5342201590538025, 3.763865679502487, 4.137318581342697, 4.187077283859253, 4.1816383600234985], + "peak_allocated_bytes": 563264512, + "incremental_peak_bytes": 333362688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09280070662498474, + "samples_ms": [0.10374188423156738, 0.09563937783241272, 0.09280070662498474, 0.09193271398544312, 0.09140744805335999], + "peak_allocated_bytes": 231267328, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8912608027458191, + "samples_ms": [1.2561529874801636, 1.1298805475234985, 0.8912608027458191, 0.8488893508911133, 0.7098987698554993], + "peak_allocated_bytes": 389049344, + "incremental_peak_bytes": 159147520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 127,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "78eb5f93f9975204fec8266c350d61fe6e0af00154590a18bff5adda82e32560", + "dtype": "torch.bfloat16", + "shape": [127, 5376] + }, + "dx": { + "sha256": "cfb4e34f9a5acfbd7f44094359cc37fb1653466897436cd847cf6c782f0fb62a", + "dtype": "torch.bfloat16", + "shape": [127, 14336] + }, + "dw": { + "sha256": "80417148de1919505236065672e995febe80f604c9acafe5c223aab71d624847", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 128, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5271827578544617e-05,"max_error_over_tolerance": 0.0003049470251426101,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000971376895904541,"max_error_over_tolerance": 0.009242621250450611,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.0865200087428093,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5223676562309265, + "samples_ms": [0.572938472032547, 0.5223676562309265, 0.5280077457427979, 0.5104504525661469, 0.5200915038585663], + "peak_allocated_bytes": 231346176, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.21956488490104675, + "samples_ms": [0.23156777024269104, 0.23876875638961792, 0.21956488490104675, 0.21862611174583435, 0.2158135175704956], + "peak_allocated_bytes": 234098688, + "incremental_peak_bytes": 4128768, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.19888579845428467, + "samples_ms": [0.20590052008628845, 0.19888579845428467, 0.20147860050201416, 0.19458308815956116, 0.1957826316356659], + "peak_allocated_bytes": 384110592, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.3782273530960083, + "samples_ms": [5.160674452781677, 3.3782273530960083, 3.2818950712680817, 3.037508577108383, 4.338420927524567], + "peak_allocated_bytes": 558444032, + "incremental_peak_bytes": 328474112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.10214373469352722, + "samples_ms": [0.11356174945831299, 0.10214373469352722, 0.09890273213386536, 0.11762231588363647, 0.10176002979278564], + "peak_allocated_bytes": 231346176, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8591338992118835, + "samples_ms": [0.8280053734779358, 0.8591338992118835, 0.5269050598144531, 0.8636452257633209, 1.0673291981220245], + "peak_allocated_bytes": 389156864, + "incremental_peak_bytes": 159186944, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 128,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5b77f873e5f379efd4a7ac6a396c7c73c851200f081d1b574928df23611350b3", + "dtype": "torch.bfloat16", + "shape": [128, 5376] + }, + "dx": { + "sha256": "3c37cb8907933fb2c047613b4ec6941d311a7fcf23484f52c90bedefcd9bf4fe", + "dtype": "torch.bfloat16", + "shape": [128, 14336] + }, + "dw": { + "sha256": "aba1da1cf968b1877fb75a15257693df005bd2fce5a66ce627eef4f30ae6ec9b", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5273690223693848e-05,"max_error_over_tolerance": 0.0003049805236514658,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009714066982269287,"max_error_over_tolerance": 0.009242905303835869,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.08660212904214859,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4708394408226013, + "samples_ms": [0.4997439682483673, 0.47484785318374634, 0.46965107321739197, 0.4708394408226013, 0.465981662273407], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2001151442527771, + "samples_ms": [0.21700188517570496, 0.204574316740036, 0.19966810941696167, 0.2001151442527771, 0.19906088709831238], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.23935362696647644, + "samples_ms": [0.2619810402393341, 0.23935362696647644, 0.23012980818748474, 0.2315528690814972, 0.24047866463661194], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.217620939016342, + "samples_ms": [4.787694662809372, 4.113547503948212, 4.1184090077877045, 4.218190908432007, 4.217620939016342], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.10640174150466919, + "samples_ms": [0.11917948722839355, 0.10800361633300781, 0.10640174150466919, 0.10486319661140442, 0.1055300235748291], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2598484754562378, + "samples_ms": [1.3252384960651398, 1.2041032314300537, 1.2598484754562378, 1.2480095028877258, 1.3608895242214203], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 256, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 2.794712781906128e-05,"max_error_over_tolerance": 0.0005570866051129997,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009754598140716553,"max_error_over_tolerance": 0.009243669919669628,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.031119346618652344,"max_error_over_tolerance": 0.11552537977695465,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5064867436885834, + "samples_ms": [0.5556009709835052, 0.5189701914787292, 0.5064867436885834, 0.4983171820640564, 0.5018599331378937], + "peak_allocated_bytes": 241442816, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22333860397338867, + "samples_ms": [0.23899972438812256, 0.22518262267112732, 0.22333860397338867, 0.22138282656669617, 0.21755322813987732], + "peak_allocated_bytes": 246947840, + "incremental_peak_bytes": 8257536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2390630543231964, + "samples_ms": [0.2594850957393646, 0.24216622114181519, 0.23720413446426392, 0.2390630543231964, 0.23523345589637756], + "peak_allocated_bytes": 392830976, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.2709968984127045, + "samples_ms": [3.2444484531879425, 8.486852049827576, 4.227753728628159, 4.548773169517517, 4.2709968984127045], + "peak_allocated_bytes": 585521664, + "incremental_peak_bytes": 346831360, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1284368336200714, + "samples_ms": [0.1284368336200714, 0.13460591435432434, 0.11734664440155029, 0.12366101145744324, 0.13120099902153015], + "peak_allocated_bytes": 241442816, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2776143848896027, + "samples_ms": [1.3336129486560822, 1.2776143848896027, 1.1073872447013855, 1.1383816599845886, 1.2937337160110474], + "peak_allocated_bytes": 402923520, + "incremental_peak_bytes": 164233216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [4,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [4,224]},"canonical_active_rows": 256,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "d0be45f3fc497360fcdb5e7dff1866a56f1703c6e1a70564917739c537e09b1e", + "dtype": "torch.bfloat16", + "shape": [256, 5376] + }, + "dx": { + "sha256": "f1a528f3d88daf90bb860d5ef2f0f42176e65c0eff3950cf880db91352ea4961", + "dtype": "torch.bfloat16", + "shape": [256, 14336] + }, + "dw": { + "sha256": "bca753e4f35b9f53e4787e16c45a1a9f61b805cc52777b45d2aac47131e792cc", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 513, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 2.748146653175354e-05,"max_error_over_tolerance": 0.0005478710518218577,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009688436985015869,"max_error_over_tolerance": 0.009219937026500702,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.031253814697265625,"max_error_over_tolerance": 0.11986027657985687,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.508233904838562, + "samples_ms": [0.5344487726688385, 0.5267933011054993, 0.508233904838562, 0.4960186779499054, 0.48770755529403687], + "peak_allocated_bytes": 261716480, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.36629289388656616, + "samples_ms": [0.38085877895355225, 0.36629289388656616, 0.3698691725730896, 0.36259740591049194, 0.36022067070007324], + "peak_allocated_bytes": 270909440, + "incremental_peak_bytes": 14708736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.3862045705318451, + "samples_ms": [0.402599573135376, 0.3907792270183563, 0.3862045705318451, 0.38257986307144165, 0.3825165331363678], + "peak_allocated_bytes": 432737792, + "incremental_peak_bytes": 176537088, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.5304372906684875, + "samples_ms": [4.907950758934021, 4.5304372906684875, 4.573829472064972, 4.422713071107864, 4.3122731149196625], + "peak_allocated_bytes": 661960704, + "incremental_peak_bytes": 405760000, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.19780173897743225, + "samples_ms": [0.19539892673492432, 0.24132058024406433, 0.19425898790359497, 0.19780173897743225, 0.30922144651412964], + "peak_allocated_bytes": 261716480, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.3085603713989258, + "samples_ms": [1.078341156244278, 1.030702143907547, 1.5124604105949402, 2.3444928228855133, 1.3085603713989258], + "peak_allocated_bytes": 430565888, + "incremental_peak_bytes": 174365184, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [9,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [9,224]},"canonical_active_rows": 513,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "e72ed22b203a375ad2b7bba287cdca983fa623986fd0409466097f088101d531", + "dtype": "torch.bfloat16", + "shape": [513, 5376] + }, + "dx": { + "sha256": "926625b6fdbe67eaaaa5ace548fe15d26850ee6ea2c3cfbfe503b42eba987e49", + "dtype": "torch.bfloat16", + "shape": [513, 14336] + }, + "dw": { + "sha256": "636678ed4d73fdd1ecab8a9b4eff1ad8eb879428c0fae9d0c8d6f9e557f886da", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 1024, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.0370429158210754e-05,"max_error_over_tolerance": 0.0006054941914044321,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009716451168060303,"max_error_over_tolerance": 0.009236982092261314,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06217002868652344,"max_error_over_tolerance": 0.14585569500923157,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.954553484916687, + "samples_ms": [1.0414011776447296, 1.0842233896255493, 0.954553484916687, 0.93870609998703, 0.9392276406288147], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.6548352539539337, + "samples_ms": [0.6848238408565521, 0.6539858877658844, 0.6239674985408783, 0.6548352539539337, 0.6782747805118561], + "peak_allocated_bytes": 320372736, + "incremental_peak_bytes": 29360128, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.6275102496147156, + "samples_ms": [0.6384477019309998, 0.6108023226261139, 0.6275102496147156, 0.6749443709850311, 0.5772635340690613], + "peak_allocated_bytes": 445153280, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.6306513249874115, + "samples_ms": [4.335585981607437, 4.6306513249874115, 4.8942118883132935, 4.492711275815964, 4.805084317922592], + "peak_allocated_bytes": 749822976, + "incremental_peak_bytes": 458810368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.2659261226654053, + "samples_ms": [0.5080997943878174, 0.2718046307563782, 0.2659261226654053, 0.26274845004081726, 0.25920569896698], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2693442404270172, + "samples_ms": [1.2693442404270172, 1.1054575443267822, 1.3971664011478424, 1.0951757431030273, 1.386839896440506], + "peak_allocated_bytes": 485523456, + "incremental_peak_bytes": 194510848, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [16,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [16,224]},"canonical_active_rows": 1024,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "183d20b577e0b0e4377a7425c17b28baf3049214b9a0c5300a3c47d9b860af5a", + "dtype": "torch.bfloat16", + "shape": [1024, 5376] + }, + "dx": { + "sha256": "33904fcbd1ec09dd48203f527771f9212638ae09c7d1a07f14fae5daad2ebec5", + "dtype": "torch.bfloat16", + "shape": [1024, 14336] + }, + "dw": { + "sha256": "afd58776763f76651a948c597d8456937d93a28295c385bb00c94228382ed5f5", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 2048, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.036297857761383e-05,"max_error_over_tolerance": 0.0006053456454537809,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009779930114746094,"max_error_over_tolerance": 0.009302103891968727,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06254959106445312,"max_error_over_tolerance": 0.14828670024871826,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.4133527874946594, + "samples_ms": [1.5480369329452515, 1.4133527874946594, 1.4082454144954681, 1.4134794473648071, 1.3939887285232544], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 1.0775364935398102, + "samples_ms": [1.0775364935398102, 1.0779686272144318, 1.081816852092743, 1.0686367750167847, 1.0738149285316467], + "peak_allocated_bytes": 419495936, + "incremental_peak_bytes": 58720256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 1.0426081717014313, + "samples_ms": [1.0580308735370636, 1.044493168592453, 1.0379701852798462, 1.0426081717014313, 1.042395830154419], + "peak_allocated_bytes": 514916352, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.988609045743942, + "samples_ms": [5.804240703582764, 6.012089550495148, 5.988609045743942, 5.955345928668976, 6.581451743841171], + "peak_allocated_bytes": 970115072, + "incremental_peak_bytes": 609339392, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.4807375371456146, + "samples_ms": [0.48383697867393494, 0.4701539874076843, 0.48117712140083313, 0.4807375371456146, 0.46753138303756714], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.5724487602710724, + "samples_ms": [1.5830844640731812, 1.5198811888694763, 1.4572776854038239, 1.6771145164966583, 1.5724487602710724], + "peak_allocated_bytes": 595656704, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [32,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [32,224]},"canonical_active_rows": 2048,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "e54e5c678f07f22f9b897444ecdc015f427004a906336492dbf908c6edd66611", + "dtype": "torch.bfloat16", + "shape": [2048, 5376] + }, + "dx": { + "sha256": "79d3974647b5f69fc8d5765d88d92f95775a5d5ae99006c5c27bbba9143f0fc8", + "dtype": "torch.bfloat16", + "shape": [2048, 14336] + }, + "dw": { + "sha256": "d3cace7e7fd59fe9eb28a725da4c1f3666f332ae5f7f167d9e60b61dd37df160", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 4096, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.0279159545898438e-05,"max_error_over_tolerance": 0.0006036892300471663,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781420230865479,"max_error_over_tolerance": 0.009296612814068794,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.11181640625,"max_error_over_tolerance": 0.14843568205833435,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 2.7626194059848785, + "samples_ms": [2.916518598794937, 2.810914069414139, 2.7626194059848785, 2.624671906232834, 2.6615671813488007], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 2.041943371295929, + "samples_ms": [2.1330080926418304, 2.076968550682068, 2.041943371295929, 1.9956156611442566, 1.956440508365631], + "peak_allocated_bytes": 617742336, + "incremental_peak_bytes": 117440512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 2.022635191679001, + "samples_ms": [2.0455382764339447, 2.022635191679001, 2.0143650472164154, 2.0569004118442535, 2.008773386478424], + "peak_allocated_bytes": 654442496, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 8.995618671178818, + "samples_ms": [9.004231542348862, 9.068693965673447, 8.96860659122467, 8.952312171459198, 8.995618671178818], + "peak_allocated_bytes": 1410699264, + "incremental_peak_bytes": 910397440, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.8829720318317413, + "samples_ms": [0.947900116443634, 0.895179808139801, 0.8829720318317413, 0.8808374404907227, 0.8783228695392609], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 2.5516562163829803, + "samples_ms": [2.5715231895446777, 2.5516562163829803, 2.6257187128067017, 2.540022134780884, 2.522353082895279], + "peak_allocated_bytes": 815923200, + "incremental_peak_bytes": 315621376, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [64,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [64,224]},"canonical_active_rows": 4096,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "1621d18fce3171753ac375d7e7d597a615918a0a51baa4631a66f856526ae3a0", + "dtype": "torch.bfloat16", + "shape": [4096, 5376] + }, + "dx": { + "sha256": "2aeba2c1c7b13af7512f076598a63022d6d4269c463c154594a6f2daadf1ef14", + "dtype": "torch.bfloat16", + "shape": [4096, 14336] + }, + "dw": { + "sha256": "223f219ee9c8d92b08f096a064bed2e0e784a20f30010e3b4c96b74a17a51110", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 8192, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.0450522899627686e-05,"max_error_over_tolerance": 0.0006070910603739321,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009778738021850586,"max_error_over_tolerance": 0.009276843629777431,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12531661987304688,"max_error_over_tolerance": 0.1686829924583435,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 6.8867504596710205, + "samples_ms": [6.461631506681442, 8.28661024570465, 6.989728659391403, 6.8867504596710205, 6.5978094935417175], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 4.373438656330109, + "samples_ms": [4.3046735227108, 4.34340164065361, 4.423592239618301, 4.373438656330109, 4.400052130222321], + "peak_allocated_bytes": 1014235136, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 4.201643168926239, + "samples_ms": [4.201643168926239, 4.208642989397049, 4.134170711040497, 4.1950903832912445, 4.259232431650162], + "peak_allocated_bytes": 933494784, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 18.23057234287262, + "samples_ms": [22.7372869849205, 19.50514316558838, 18.23057234287262, 18.210425972938538, 17.958156764507294], + "peak_allocated_bytes": 2291867648, + "incremental_peak_bytes": 1512513536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 1.7067305743694305, + "samples_ms": [1.721072942018509, 1.7067305743694305, 1.7101801931858063, 1.706693321466446, 1.7048045992851257], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 5.713455379009247, + "samples_ms": [5.423523485660553, 5.721472203731537, 5.715426057577133, 5.713455379009247, 5.707930773496628], + "peak_allocated_bytes": 1256456192, + "incremental_peak_bytes": 477102080, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [128,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [128,224]},"canonical_active_rows": 8192,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6a54a53cc4462816868b87f3f7a5d5ac1748a1de999f1904a2d3e9d7414644dc", + "dtype": "torch.bfloat16", + "shape": [8192, 5376] + }, + "dx": { + "sha256": "caedce8f925825f99fe5013c75f3c840718bf79cf9c680cf55314447f6b088df", + "dtype": "torch.bfloat16", + "shape": [8192, 14336] + }, + "dw": { + "sha256": "941d3b7ded8cbec8ede620b452d917b70c68b286c690e7640d206d66e24352cc", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 16385, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.045797348022461e-05,"max_error_over_tolerance": 0.0006071952520869672,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009784698486328125,"max_error_over_tolerance": 0.009305674582719803,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.2373809814453125,"max_error_over_tolerance": 0.16933324933052063,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 20.00322937965393, + "samples_ms": [20.033571869134903, 20.11100947856903, 19.97118815779686, 20.00322937965393, 19.972704350948334], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 8.884463459253311, + "samples_ms": [8.922353386878967, 8.921671658754349, 8.82551446557045, 8.884463459253311, 8.845862001180649], + "peak_allocated_bytes": 1807319040, + "incremental_peak_bytes": 469790720, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 9.360156953334808, + "samples_ms": [9.191662073135376, 9.159374982118607, 9.420003741979599, 9.573694318532944, 9.360156953334808], + "peak_allocated_bytes": 2138853376, + "incremental_peak_bytes": 801325056, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 43.33186149597168, + "samples_ms": [46.373505145311356, 42.94878989458084, 43.254923075437546, 44.764868915081024, 43.33186149597168], + "peak_allocated_bytes": 4701607936, + "incremental_peak_bytes": 3364079616, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 3.1894929707050323, + "samples_ms": [3.1894929707050323, 3.1828992068767548, 3.18700447678566, 3.405846655368805, 3.1903572380542755], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 11.646967381238937, + "samples_ms": [11.43326610326767, 11.646967381238937, 11.721748858690262, 11.743105947971344, 11.5024633705616], + "peak_allocated_bytes": 2137631232, + "incremental_peak_bytes": 800102912, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [257,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [257,224]},"canonical_active_rows": 16385,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "aa4bf4200f549127eb85c3a288b7224d4a16b52a8043013d29d71f7be73284ef", + "dtype": "torch.bfloat16", + "shape": [16385, 5376] + }, + "dx": { + "sha256": "8cd2308514c54aaadb5af89decef441854e45b7ba8b199e8acc5949f4c754650", + "dtype": "torch.bfloat16", + "shape": [16385, 14336] + }, + "dw": { + "sha256": "430f8d5f6e7a36da69ea70998e41ade26ff44b9406cd9fe0f7d12222c4d74167", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 24577, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.0390918254852295e-05,"max_error_over_tolerance": 0.0006057700957171619,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009780526161193848,"max_error_over_tolerance": 0.009305845014750957,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.25164031982421875,"max_error_over_tolerance": 0.18054932355880737,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 31.06134384870529, + "samples_ms": [31.1649888753891, 31.079281121492386, 30.950598418712616, 30.888359993696213, 31.06134384870529], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 18.26496049761772, + "samples_ms": [18.45216378569603, 18.024198710918427, 17.759855836629868, 19.124291837215424, 18.26496049761772], + "peak_allocated_bytes": 2600304640, + "incremental_peak_bytes": 704671744, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 14.54518735408783, + "samples_ms": [14.171525835990906, 14.548677951097488, 14.515917748212814, 14.54518735408783, 14.740563929080963], + "peak_allocated_bytes": 3019919360, + "incremental_peak_bytes": 1124286464, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 71.44178822636604, + "samples_ms": [71.44178822636604, 70.59933245182037, 71.16098329424858, 73.56076687574387, 73.6989676952362], + "peak_allocated_bytes": 6786906112, + "incremental_peak_bytes": 4891273216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 4.727460443973541, + "samples_ms": [4.791069775819778, 4.766747355461121, 4.727460443973541, 4.718884825706482, 4.720926284790039], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 18.98377016186714, + "samples_ms": [19.223317503929138, 18.994607031345367, 18.97561177611351, 18.98377016186714, 18.50433275103569], + "peak_allocated_bytes": 3018697216, + "incremental_peak_bytes": 1123064320, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [385,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [385,224]},"canonical_active_rows": 24577,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "4a4ae7db6de8e3bb9ea488f2da53afd2fcdd526febfc27e20f32db1696112375", + "dtype": "torch.bfloat16", + "shape": [24577, 5376] + }, + "dx": { + "sha256": "fd4a3d555ad9b8e8a0be0a0ac212d337c20e4fd6e691ce1982b341ceb2818616", + "dtype": "torch.bfloat16", + "shape": [24577, 14336] + }, + "dw": { + "sha256": "facb4ce8b0cb9c90f9451956631e2490da927d2a8ec36bf07eaf6b5c237f6cfe", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32768, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 3.0469149351119995e-05,"max_error_over_tolerance": 0.0006074772099964321,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781122207641602,"max_error_over_tolerance": 0.009311916306614876,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.252532958984375,"max_error_over_tolerance": 0.18232759833335876,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 41.69951379299164, + "samples_ms": [41.7838990688324, 41.689883917570114, 41.70183464884758, 41.69951379299164, 41.60373657941818], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 36.576222628355026, + "samples_ms": [37.16915473341942, 36.50326281785965, 36.96649894118309, 36.576222628355026, 34.471482038497925], + "peak_allocated_bytes": 3393191936, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 18.744267523288727, + "samples_ms": [18.744267523288727, 20.046759396791458, 18.29393580555916, 18.44470575451851, 19.663449376821518], + "peak_allocated_bytes": 2607808512, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 106.80445283651352, + "samples_ms": [108.48763585090637, 104.52854260802269, 109.95243117213249, 106.80445283651352, 105.52418604493141], + "peak_allocated_bytes": 7578877952, + "incremental_peak_bytes": 5125210112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 6.743166595697403, + "samples_ms": [6.233599036931992, 6.239496171474457, 6.743166595697403, 7.567781955003738, 7.592573761940002], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 20.594339817762375, + "samples_ms": [22.5556418299675, 21.25835418701172, 20.080365240573883, 20.07279545068741, 20.594339817762375], + "peak_allocated_bytes": 3899654144, + "incremental_peak_bytes": 1445986304, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [512,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [512,224]},"canonical_active_rows": 32768,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "fb0ebbf1abddf063f48ef33b3af3473a1c6e77dd25c71e53436c36bc27bf5633", + "dtype": "torch.bfloat16", + "shape": [32768, 5376] + }, + "dx": { + "sha256": "5500147da973aec9b668136996f3cf618885a6a54a3293528cd0c5b8382c6f59", + "dtype": "torch.bfloat16", + "shape": [32768, 14336] + }, + "dw": { + "sha256": "4fa5b8c06544657f604f5ccfeca21afbae4284d9ed941aef981c0098a0c7b7ca", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + } + ] +} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-random.json b/docs/operators/evidence/h3-down-h100-20261010/h100-random.json new file mode 100644 index 000000000..0771f4ea7 --- /dev/null +++ b/docs/operators/evidence/h3-down-h100-20261010/h100-random.json @@ -0,0 +1,1516 @@ +{ + "schema": "h3-down-measured-case-part-v1", + "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", + "case_indexes": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + "cases": [ + { + "rows": 1, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0077440738677978516,"max_error_over_tolerance": 0.08342820405960083,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000848233699798584,"max_error_over_tolerance": 0.008046787232160568,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.0324886292219162,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4641115665435791, + "samples_ms": [0.5014799535274506, 0.4641115665435791, 0.4806891083717346, 0.46247243881225586, 0.46306103467941284], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19700080156326294, + "samples_ms": [0.19984692335128784, 0.21108612418174744, 0.19700080156326294, 0.18755346536636353, 0.18952041864395142], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.16470998525619507, + "samples_ms": [0.18192827701568604, 0.1676119863986969, 0.16402080655097961, 0.16470998525619507, 0.1600421965122223], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.881074815988541, + "samples_ms": [10.020162910223007, 4.396550357341766, 4.599045962095261, 5.881074815988541, 7.008261978626251], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08215755224227905, + "samples_ms": [0.107593834400177, 0.08467957377433777, 0.08215755224227905, 0.07981061935424805, 0.07973238825798035], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2091360986232758, + "samples_ms": [0.8774884045124054, 1.2091360986232758, 1.040332019329071, 1.277294009923935, 1.5825442969799042], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 31, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014126777648925781,"max_error_over_tolerance": 0.10740139335393906,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009508132934570312,"max_error_over_tolerance": 0.009040186181664467,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007812023162841797,"max_error_over_tolerance": 0.055731140077114105,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6497353315353394, + "samples_ms": [0.6950311362743378, 0.6497353315353394, 0.6531104445457458, 0.6261691451072693, 0.6205104291439056], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.28938427567481995, + "samples_ms": [0.31585991382598877, 0.29233843088150024, 0.2748854458332062, 0.2711452543735504, 0.28938427567481995], + "peak_allocated_bytes": 224250880, + "incremental_peak_bytes": 888832, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.3261081874370575, + "samples_ms": [0.35964325070381165, 0.3442540764808655, 0.31934306025505066, 0.3134310245513916, 0.3261081874370575], + "peak_allocated_bytes": 378764288, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.858814179897308, + "samples_ms": [4.203196614980698, 4.415713250637054, 4.858814179897308, 6.295427680015564, 5.825243890285492], + "peak_allocated_bytes": 537462784, + "incremental_peak_bytes": 314100736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1394413411617279, + "samples_ms": [0.1394413411617279, 3.767944872379303, 0.316668301820755, 0.11179596185684204, 0.10490044951438904], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.3986565172672272, + "samples_ms": [1.3986565172672272, 1.439116895198822, 1.512467861175537, 1.0717213153839111, 0.9977258741855621], + "peak_allocated_bytes": 378724864, + "incremental_peak_bytes": 155362816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 31,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "bc382a772a452fbea1b5ee787fb4e0b65064e260c534136fbb25491a328f9d86", + "dtype": "torch.bfloat16", + "shape": [31, 5376] + }, + "dx": { + "sha256": "6266236128cb9315856b07e8bb3b991103c0067b4ceead50206f91bb9eab6975", + "dtype": "torch.bfloat16", + "shape": [31, 14336] + }, + "dw": { + "sha256": "fab683c8375ddd2d5031ab580471c42c5cb0194aeda91773d51337366aa21321", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014126777648925781,"max_error_over_tolerance": 0.10740139335393906,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009508132934570312,"max_error_over_tolerance": 0.009040186181664467,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007812738418579102,"max_error_over_tolerance": 0.05573455989360809,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.48016756772994995, + "samples_ms": [0.5154013633728027, 0.49382075667381287, 0.48016756772994995, 0.4757493734359741, 0.4628002643585205], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19780173897743225, + "samples_ms": [0.21325424313545227, 0.20276755094528198, 0.19780173897743225, 0.19660592079162598, 0.19564852118492126], + "peak_allocated_bytes": 224347648, + "incremental_peak_bytes": 917504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.14327839016914368, + "samples_ms": [0.15502050518989563, 0.14247745275497437, 0.14327839016914368, 0.1337788999080658, 0.14632940292358398], + "peak_allocated_bytes": 377570816, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.488680511713028, + "samples_ms": [4.308421164751053, 4.495076835155487, 5.520697683095932, 4.234891384840012, 4.488680511713028], + "peak_allocated_bytes": 536416256, + "incremental_peak_bytes": 312986112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09065121412277222, + "samples_ms": [0.10181963443756104, 0.09404495358467102, 0.09065121412277222, 0.08993595838546753, 0.08934363722801208], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.9762495756149292, + "samples_ms": [1.1316165328025818, 0.9762495756149292, 1.1035725474357605, 0.894542783498764, 0.8411481976509094], + "peak_allocated_bytes": 378832384, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 32,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "b5d3d5e2880a7ce138593c82d8e6f7903f4b6eee7e6fea9c4347ff0e13819f71", + "dtype": "torch.bfloat16", + "shape": [32, 5376] + }, + "dx": { + "sha256": "d7424a7f439bb3784a9fd0c29fdfed30911cc028598d1763906f4ede9c700fdc", + "dtype": "torch.bfloat16", + "shape": [32, 14336] + }, + "dw": { + "sha256": "5afe57d24a47021447aea4bdb28d19646b771e50129c64b65b697673ba4e66d7", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014126300811767578,"max_error_over_tolerance": 0.10739777982234955,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009508132934570312,"max_error_over_tolerance": 0.009040186181664467,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.055741358548402786,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5179569125175476, + "samples_ms": [0.5485452711582184, 0.5214326083660126, 0.5179569125175476, 0.5115382373332977, 0.5022026598453522], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2129785716533661, + "samples_ms": [0.2353973686695099, 0.22062286734580994, 0.2129785716533661, 0.2116560935974121, 0.20866096019744873], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.22759661078453064, + "samples_ms": [0.24954602122306824, 0.22759661078453064, 0.22070109844207764, 0.23080036044120789, 0.22421404719352722], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.993552178144455, + "samples_ms": [3.993552178144455, 3.9886869490146637, 4.171747714281082, 3.9090216159820557, 4.432704299688339], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08925795555114746, + "samples_ms": [0.09898096323013306, 0.09119883179664612, 0.08878856897354126, 0.08925795555114746, 0.08856505155563354], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.4379099011421204, + "samples_ms": [5.710933357477188, 2.139914780855179, 1.4379099011421204, 0.9647868573665619, 1.1961422860622406], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 127, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015645503997802734,"max_error_over_tolerance": 0.12006104737520218,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.0866260752081871,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5102045834064484, + "samples_ms": [0.5356669425964355, 0.5090311169624329, 0.4749707877635956, 0.5102045834064484, 0.5424767732620239], + "peak_allocated_bytes": 231267328, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22759288549423218, + "samples_ms": [0.2511031925678253, 0.2266913652420044, 0.2282075583934784, 0.22707507014274597, 0.22759288549423218], + "peak_allocated_bytes": 233998336, + "incremental_peak_bytes": 4096512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.21474435925483704, + "samples_ms": [0.23312494158744812, 0.21474435925483704, 0.21308287978172302, 0.2142675220966339, 0.2564825117588043], + "peak_allocated_bytes": 389515264, + "incremental_peak_bytes": 159613440, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.306524991989136, + "samples_ms": [4.206404089927673, 3.9475373923778534, 4.449903964996338, 11.210303753614426, 4.306524991989136], + "peak_allocated_bytes": 563264512, + "incremental_peak_bytes": 333362688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09214133024215698, + "samples_ms": [0.11442974209785461, 0.09933486580848694, 0.09214133024215698, 0.09178370237350464, 0.09139999747276306], + "peak_allocated_bytes": 231267328, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.6517879664897919, + "samples_ms": [0.8602924644947052, 0.738535076379776, 0.6517879664897919, 0.4775114357471466, 0.48815831542015076], + "peak_allocated_bytes": 389049344, + "incremental_peak_bytes": 159147520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 127,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "7ba29d94e89afc0b879e4f3a22d64b0d62442a747d28d4c913f7fe6456fbf347", + "dtype": "torch.bfloat16", + "shape": [127, 5376] + }, + "dx": { + "sha256": "296804066060eabb8d1b14ed7dbc3e4ecbb2315955292acf1c08f818fcef32c5", + "dtype": "torch.bfloat16", + "shape": [127, 14336] + }, + "dw": { + "sha256": "de7318bed0fb5661d902180de7980b37e9a6cc78f8a6bfa5e098613fe80a7dcb", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 128, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015645503997802734,"max_error_over_tolerance": 0.12006104737520218,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.0865200087428093,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5424097180366516, + "samples_ms": [0.5952082574367523, 0.5424097180366516, 0.5490556359291077, 0.5400031805038452, 0.5214028060436249], + "peak_allocated_bytes": 231346176, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22576004266738892, + "samples_ms": [0.25213882327079773, 0.2295970916748047, 0.22279098629951477, 0.22278353571891785, 0.22576004266738892], + "peak_allocated_bytes": 234098688, + "incremental_peak_bytes": 4128768, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.210687518119812, + "samples_ms": [0.22293627262115479, 0.210687518119812, 0.20804256200790405, 0.20571425557136536, 0.21763518452644348], + "peak_allocated_bytes": 384110592, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.423797130584717, + "samples_ms": [3.662385046482086, 4.423797130584717, 4.459649324417114, 4.2327046394348145, 4.465457051992416], + "peak_allocated_bytes": 558444032, + "incremental_peak_bytes": 328474112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09127333760261536, + "samples_ms": [0.121239572763443, 0.09503215551376343, 0.09127333760261536, 0.09118765592575073, 0.0912398099899292], + "peak_allocated_bytes": 231346176, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.309465616941452, + "samples_ms": [1.0634250938892365, 1.309465616941452, 1.2851320207118988, 1.330547034740448, 1.3811662793159485], + "peak_allocated_bytes": 389156864, + "incremental_peak_bytes": 159186944, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 128,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6407e4fc7b31d0fb6bc7983e9b73af0ed6865d38a996701f28dd5b96653b4316", + "dtype": "torch.bfloat16", + "shape": [128, 5376] + }, + "dx": { + "sha256": "0a8d2486c11b7036a22b52461a48637df6db771a88ca37c15917e3497c33195e", + "dtype": "torch.bfloat16", + "shape": [128, 14336] + }, + "dw": { + "sha256": "97560750faf93915f48fff41bf45f3f36d5544ee992bc04f79a7a54613bd477c", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01564645767211914,"max_error_over_tolerance": 0.12006834149360657,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625476837158203,"max_error_over_tolerance": 0.08660759031772614,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5118474364280701, + "samples_ms": [0.5456544458866119, 0.5125962197780609, 0.5012191832065582, 0.5118474364280701, 0.5014054477214813], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22323429584503174, + "samples_ms": [0.2366490662097931, 0.22323429584503174, 0.21626055240631104, 0.22421032190322876, 0.21079927682876587], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2612210810184479, + "samples_ms": [0.29610469937324524, 0.2780333161354065, 0.25999173521995544, 0.2612210810184479, 0.24727359414100647], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.367075860500336, + "samples_ms": [9.567219763994217, 4.193499684333801, 4.239816218614578, 4.367075860500336, 4.3790265917778015], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11000409722328186, + "samples_ms": [0.122804194688797, 0.11217594146728516, 0.10993704199790955, 0.10891258716583252, 0.11000409722328186], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.257181167602539, + "samples_ms": [1.0933689773082733, 1.257181167602539, 1.3633482158184052, 1.405365765094757, 0.9644664824008942], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 256, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015221118927001953,"max_error_over_tolerance": 0.11568786203861237,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009686648845672607,"max_error_over_tolerance": 0.009199724532663822,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.031119346618652344,"max_error_over_tolerance": 0.11714421957731247,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4711635410785675, + "samples_ms": [0.49818307161331177, 0.4711635410785675, 0.4619285464286804, 0.4760324954986572, 0.46869367361068726], + "peak_allocated_bytes": 241442816, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.23234263062477112, + "samples_ms": [0.2589598298072815, 0.23838505148887634, 0.2281591296195984, 0.23234263062477112, 0.22763758897781372], + "peak_allocated_bytes": 246947840, + "incremental_peak_bytes": 8257536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2569854259490967, + "samples_ms": [0.27580931782722473, 0.2569854259490967, 0.26258453726768494, 0.2506449818611145, 0.24703145027160645], + "peak_allocated_bytes": 392830976, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.024874418973923, + "samples_ms": [3.8254931569099426, 4.024874418973923, 4.399307072162628, 4.058755934238434, 3.2460279762744904], + "peak_allocated_bytes": 585521664, + "incremental_peak_bytes": 346831360, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1119375228881836, + "samples_ms": [0.12212246656417847, 0.11330097913742065, 0.11178478598594666, 0.1119375228881836, 0.11095404624938965], + "peak_allocated_bytes": 241442816, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.44563785195350647, + "samples_ms": [0.8560195565223694, 0.44563785195350647, 0.43587759137153625, 1.0146237909793854, 0.43319910764694214], + "peak_allocated_bytes": 402923520, + "incremental_peak_bytes": 164233216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [4,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [4,224]},"canonical_active_rows": 256,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "3659607138418e29f2f698deae3d17b367d227d3c6ea67dffa049e59a3feb8f4", + "dtype": "torch.bfloat16", + "shape": [256, 5376] + }, + "dx": { + "sha256": "8e6c64afeb2a469c4de837c5e087a30bf3c3cc619e78e45726feadd328c33b5f", + "dtype": "torch.bfloat16", + "shape": [256, 14336] + }, + "dw": { + "sha256": "f407bc90aa87075f4470ac7fac09940e27cbd0004a6c3b1f8967a811dd95d358", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 513, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015606880187988281,"max_error_over_tolerance": 0.11976537108421326,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009740591049194336,"max_error_over_tolerance": 0.009257806465029716,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.03125476837158203,"max_error_over_tolerance": 0.11986027657985687,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5597323179244995, + "samples_ms": [0.6031058728694916, 0.5706027150154114, 0.5584210157394409, 0.5597323179244995, 0.5541481077671051], + "peak_allocated_bytes": 261716480, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.3941841423511505, + "samples_ms": [0.4095993936061859, 0.401180237531662, 0.3921687602996826, 0.3941841423511505, 0.3767870366573334], + "peak_allocated_bytes": 270909440, + "incremental_peak_bytes": 14708736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.4532076418399811, + "samples_ms": [0.4839673638343811, 0.4532076418399811, 0.4369504749774933, 0.46051666140556335, 0.43874233961105347], + "peak_allocated_bytes": 432737792, + "incremental_peak_bytes": 176537088, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.67793270945549, + "samples_ms": [4.385463893413544, 4.415247589349747, 4.6799443662166595, 4.7153085470199585, 4.67793270945549], + "peak_allocated_bytes": 661960704, + "incremental_peak_bytes": 405760000, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.20292028784751892, + "samples_ms": [0.20292028784751892, 0.19189715385437012, 0.20329654216766357, 0.2216733992099762, 0.18816441297531128], + "peak_allocated_bytes": 261716480, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.4380626380443573, + "samples_ms": [1.453697681427002, 1.389387995004654, 1.4380626380443573, 1.496635377407074, 1.407429575920105], + "peak_allocated_bytes": 430565888, + "incremental_peak_bytes": 174365184, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [9,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [9,224]},"canonical_active_rows": 513,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "3a6f35aed6e271d33e0a7cc5acd38e4ab30af9c34f441f93ca727ca7f90b9338", + "dtype": "torch.bfloat16", + "shape": [513, 5376] + }, + "dx": { + "sha256": "6695834948f5dc214db7e6893ad879700193dc7e6bc82bcb42a9954b7f60a609", + "dtype": "torch.bfloat16", + "shape": [513, 14336] + }, + "dw": { + "sha256": "cc181cb9f40cbe14e4b49f08e7a81ca61fbbe6f97fd291241596085501f961cd", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 1024, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015606880187988281,"max_error_over_tolerance": 0.11976537108421326,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009763240814208984,"max_error_over_tolerance": 0.009276159107685089,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06217002868652344,"max_error_over_tolerance": 0.14585569500923157,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.0156221687793732, + "samples_ms": [1.071583479642868, 1.0156221687793732, 1.0580532252788544, 0.9893365204334259, 0.9548701345920563], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.6798282265663147, + "samples_ms": [0.686943531036377, 0.6731189787387848, 0.6798282265663147, 0.697501003742218, 0.6656236946582794], + "peak_allocated_bytes": 320372736, + "incremental_peak_bytes": 29360128, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.5863979458808899, + "samples_ms": [0.6058663129806519, 0.5968138575553894, 0.5863979458808899, 0.580664724111557, 0.5841217935085297], + "peak_allocated_bytes": 445153280, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.7401003539562225, + "samples_ms": [4.143659025430679, 4.2868852615356445, 4.7401003539562225, 4.927046597003937, 12.686174362897873], + "peak_allocated_bytes": 749822976, + "incremental_peak_bytes": 458810368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.2707056701183319, + "samples_ms": [0.2880282700061798, 0.2771615982055664, 0.2707056701183319, 0.26961416006088257, 0.26987865567207336], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.7799342274665833, + "samples_ms": [0.8296966552734375, 0.7799342274665833, 0.7755793631076813, 0.7859431207180023, 0.7728338241577148], + "peak_allocated_bytes": 485523456, + "incremental_peak_bytes": 194510848, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [16,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [16,224]},"canonical_active_rows": 1024,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "63b5882d73e482b20dcdba60ad939e2adf576ad4a6deed18640700a874292db2", + "dtype": "torch.bfloat16", + "shape": [1024, 5376] + }, + "dx": { + "sha256": "75f218d54abf82a3e40b1e6d7c4a9b7301826c7680cfa714679a0dd9df69b7a5", + "dtype": "torch.bfloat16", + "shape": [1024, 14336] + }, + "dw": { + "sha256": "0f7c2b049a78f21d8103882afc0b7de005c54445911cc0de881766436fbf48e7", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 2048, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01564931869506836,"max_error_over_tolerance": 0.11999168992042542,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009767413139343262,"max_error_over_tolerance": 0.00929025188088417,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06254959106445312,"max_error_over_tolerance": 0.14829984307289124,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.5788264572620392, + "samples_ms": [1.6528479754924774, 1.5788264572620392, 1.5440359711647034, 1.575414091348648, 2.727128565311432], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 1.134667545557022, + "samples_ms": [1.8423348665237427, 1.1614300310611725, 1.1234171688556671, 1.134667545557022, 1.126706600189209], + "peak_allocated_bytes": 419495936, + "incremental_peak_bytes": 58720256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 1.0633394122123718, + "samples_ms": [1.082349568605423, 1.064099371433258, 1.0633394122123718, 1.0608956217765808, 1.055743545293808], + "peak_allocated_bytes": 514916352, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 6.045274436473846, + "samples_ms": [5.990024656057358, 6.045274436473846, 6.000753492116928, 6.2157511711120605, 6.278932094573975], + "peak_allocated_bytes": 970115072, + "incremental_peak_bytes": 609339392, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.5051419138908386, + "samples_ms": [0.5087107419967651, 0.5051419138908386, 0.48294663429260254, 0.5096010863780975, 0.4931837320327759], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.4390572905540466, + "samples_ms": [1.4390572905540466, 1.6509965062141418, 1.4427416026592255, 1.3899244368076324, 1.398913562297821], + "peak_allocated_bytes": 595656704, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [32,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [32,224]},"canonical_active_rows": 2048,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "248d6c0ce328417c18509b2e02a46d8227a6d94fc870be78561c534b7151c9f5", + "dtype": "torch.bfloat16", + "shape": [2048, 5376] + }, + "dx": { + "sha256": "cbfbca4476a162e57b24cffd7aab8eed3e7b295a7d34bb6b2364fabf12132b07", + "dtype": "torch.bfloat16", + "shape": [2048, 14336] + }, + "dw": { + "sha256": "660d931eee7dfe0d3da99d6052a9bd97a484ec62633b5fc358ecba64f30c4103", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 4096, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015695571899414062,"max_error_over_tolerance": 0.1199989914894104,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781420230865479,"max_error_over_tolerance": 0.009296612814068794,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.1242218017578125,"max_error_over_tolerance": 0.167305588722229,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 2.7483180165290833, + "samples_ms": [2.8183721005916595, 2.7756132185459137, 2.7483180165290833, 2.717547118663788, 2.71620973944664], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 2.094157040119171, + "samples_ms": [2.1256916224956512, 2.094157040119171, 2.104472368955612, 2.067010849714279, 2.069532871246338], + "peak_allocated_bytes": 617742336, + "incremental_peak_bytes": 117440512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 2.0535029470920563, + "samples_ms": [2.072621136903763, 2.0535029470920563, 1.989707350730896, 2.037379890680313, 2.0740963518619537], + "peak_allocated_bytes": 654442496, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 9.550560265779495, + "samples_ms": [9.37473401427269, 9.506858885288239, 9.550560265779495, 9.798500686883926, 9.749151766300201], + "peak_allocated_bytes": 1410699264, + "incremental_peak_bytes": 910397440, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.8845888078212738, + "samples_ms": [0.9028799831867218, 0.8845888078212738, 0.8811093866825104, 0.8833743631839752, 0.8864738047122955], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 2.771172672510147, + "samples_ms": [2.709783613681793, 2.771172672510147, 2.711579203605652, 8.380550891160965, 5.2012912929058075], + "peak_allocated_bytes": 815923200, + "incremental_peak_bytes": 315621376, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [64,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [64,224]},"canonical_active_rows": 4096,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "8953d5c75afec1f337c637dd11f465879f3927856f98fbd5510996f20a07e4c7", + "dtype": "torch.bfloat16", + "shape": [4096, 5376] + }, + "dx": { + "sha256": "fd023c1794371e2b1d28e1f1f534b585d47d3200b701103a0cdfecf11a5687ad", + "dtype": "torch.bfloat16", + "shape": [4096, 14336] + }, + "dw": { + "sha256": "dfd59a6bfc69e8afe7672c2e883f2374938802dd8ea6e4be49c3df10b6e6e6bc", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 8192, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015656471252441406,"max_error_over_tolerance": 0.11954245716333389,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000977635383605957,"max_error_over_tolerance": 0.009301135316491127,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12531661987304688,"max_error_over_tolerance": 0.1686829924583435,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 7.1127749979496, + "samples_ms": [7.472507655620575, 7.3508769273757935, 7.102977484464645, 7.1127749979496, 6.655432283878326], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 4.442032426595688, + "samples_ms": [4.521660506725311, 4.305578768253326, 4.4516921043396, 4.442032426595688, 4.309028387069702], + "peak_allocated_bytes": 1014235136, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 4.242591559886932, + "samples_ms": [4.133425652980804, 4.224028438329697, 4.242591559886932, 4.332114011049271, 4.297386854887009], + "peak_allocated_bytes": 933494784, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 18.447231501340866, + "samples_ms": [18.785148859024048, 18.268324434757233, 18.301360309123993, 19.21449601650238, 18.447231501340866], + "peak_allocated_bytes": 2291867648, + "incremental_peak_bytes": 1512513536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 1.7193928360939026, + "samples_ms": [1.7297640442848206, 1.718033105134964, 1.7193928360939026, 1.7182528972625732, 1.7548836767673492], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 5.222029983997345, + "samples_ms": [5.2265264093875885, 5.222029983997345, 5.225643515586853, 5.210563540458679, 5.208142101764679], + "peak_allocated_bytes": 1256456192, + "incremental_peak_bytes": 477102080, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [128,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [128,224]},"canonical_active_rows": 8192,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8", + "dtype": "torch.bfloat16", + "shape": [8192, 5376] + }, + "dx": { + "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66", + "dtype": "torch.bfloat16", + "shape": [8192, 14336] + }, + "dw": { + "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 16385, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015697956085205078,"max_error_over_tolerance": 0.11958968639373779,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000978320837020874,"max_error_over_tolerance": 0.00930879544466734,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.2373809814453125,"max_error_over_tolerance": 0.16933324933052063,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 19.976794719696045, + "samples_ms": [20.11549100279808, 20.03374695777893, 19.976794719696045, 19.92051675915718, 19.936978816986084], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 8.871227502822876, + "samples_ms": [8.871227502822876, 8.881252259016037, 8.826285600662231, 8.820831775665283, 8.926533162593842], + "peak_allocated_bytes": 1807319040, + "incremental_peak_bytes": 469790720, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 9.569797664880753, + "samples_ms": [9.343385696411133, 9.544700384140015, 9.606815874576569, 9.950287640094757, 9.569797664880753], + "peak_allocated_bytes": 2138853376, + "incremental_peak_bytes": 801325056, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 43.10831427574158, + "samples_ms": [43.07585582137108, 43.23599115014076, 45.22192105650902, 43.10831427574158, 43.03518310189247], + "peak_allocated_bytes": 4701607936, + "incremental_peak_bytes": 3364079616, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 3.1672269105911255, + "samples_ms": [3.1685233116149902, 3.171522170305252, 3.1575746834278107, 3.1672269105911255, 3.161471337080002], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 11.300873011350632, + "samples_ms": [11.613503098487854, 12.109711766242981, 11.300873011350632, 11.228453367948532, 11.10968366265297], + "peak_allocated_bytes": 2137631232, + "incremental_peak_bytes": 800102912, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [257,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [257,224]},"canonical_active_rows": 16385,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "c4a1b1c7e38ae17487e7ef2238f3c3f2bd323df574ffb9996432a3598df29ace", + "dtype": "torch.bfloat16", + "shape": [16385, 5376] + }, + "dx": { + "sha256": "bc96cb96333511e4ecf5dbc95fbe5beab1bf830f2d9cf4cee004c2fce6340052", + "dtype": "torch.bfloat16", + "shape": [16385, 14336] + }, + "dw": { + "sha256": "a30ac927fbd5c9d1222c651d76e0fd188e336b7240d74d71f27407e2013c1cbf", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 24577, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015681743621826172,"max_error_over_tolerance": 0.12001358717679977,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009780824184417725,"max_error_over_tolerance": 0.009305674582719803,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.25164031982421875,"max_error_over_tolerance": 0.18054932355880737,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 30.980128794908524, + "samples_ms": [30.980128794908524, 31.55752643942833, 32.05243498086929, 30.90331330895424, 30.965466052293777], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 18.71088147163391, + "samples_ms": [20.349707454442978, 19.794635474681854, 18.71088147163391, 16.914289444684982, 16.702190041542053], + "peak_allocated_bytes": 2600304640, + "incremental_peak_bytes": 704671744, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 14.257971197366714, + "samples_ms": [14.178819954395294, 13.901382684707642, 14.257971197366714, 16.387518495321274, 16.816169023513794], + "peak_allocated_bytes": 3019919360, + "incremental_peak_bytes": 1124286464, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 72.33802601695061, + "samples_ms": [70.44867426156998, 72.74215668439865, 73.1935054063797, 72.33802601695061, 69.4698803126812], + "peak_allocated_bytes": 6786906112, + "incremental_peak_bytes": 4891273216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 4.81795147061348, + "samples_ms": [4.900552332401276, 4.775349050760269, 4.790332168340683, 4.81795147061348, 4.820369184017181], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 18.448282033205032, + "samples_ms": [16.797229647636414, 17.87669211626053, 18.863487988710403, 18.856793642044067, 18.448282033205032], + "peak_allocated_bytes": 3018697216, + "incremental_peak_bytes": 1123064320, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [385,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [385,224]},"canonical_active_rows": 24577,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "36be331ffee01449374ae65b90be093f4f859b8d8e2e4a181e6e280375a4e0b9", + "dtype": "torch.bfloat16", + "shape": [24577, 5376] + }, + "dx": { + "sha256": "f407c423a0726b739f5470ddd7ea438cb2532e9482f4a69f6f64853fdc2bf62c", + "dtype": "torch.bfloat16", + "shape": [24577, 14336] + }, + "dw": { + "sha256": "1b384c414dc742aee13cd82e76a9a3824d8f8287a7821a64116e98240500cbfd", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32768, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01569652557373047,"max_error_over_tolerance": 0.12013405561447144,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781122207641602,"max_error_over_tolerance": 0.009311916306614876,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.2525482177734375,"max_error_over_tolerance": 0.18232759833335876,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 41.62594676017761, + "samples_ms": [41.61025583744049, 41.62594676017761, 41.63192957639694, 41.6228361427784, 41.692305356264114], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 35.47533228993416, + "samples_ms": [33.461909741163254, 34.31571647524834, 37.22900524735451, 35.47533228993416, 35.92732176184654], + "peak_allocated_bytes": 3393191936, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 19.52551305294037, + "samples_ms": [19.52551305294037, 19.28829774260521, 17.499323934316635, 20.080339163541794, 20.596228539943695], + "peak_allocated_bytes": 2607808512, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 107.59353637695312, + "samples_ms": [109.32975634932518, 107.23195970058441, 106.83485865592957, 107.59353637695312, 112.38069087266922], + "peak_allocated_bytes": 7578877952, + "incremental_peak_bytes": 5125210112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 7.167089730501175, + "samples_ms": [6.37710839509964, 6.479129195213318, 7.427748292684555, 7.846798747777939, 7.167089730501175], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 21.537311375141144, + "samples_ms": [22.03044295310974, 21.537311375141144, 20.823486149311066, 21.31655439734459, 21.96405827999115], + "peak_allocated_bytes": 3899654144, + "incremental_peak_bytes": 1445986304, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [512,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [512,224]},"canonical_active_rows": 32768,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949", + "dtype": "torch.bfloat16", + "shape": [32768, 5376] + }, + "dx": { + "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515", + "dtype": "torch.bfloat16", + "shape": [32768, 14336] + }, + "dw": { + "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + } + ] +} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json b/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json new file mode 100644 index 000000000..b8dce1774 --- /dev/null +++ b/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json @@ -0,0 +1,2118 @@ +{ + "schema": "h3-down-measured-replays-v1", + "cold_cache": { + "schema": "h3-down-qualification-v2", + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "status": "passed", + "cases": [ + { + "rows": 1, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0077440738677978516,"max_error_over_tolerance": 0.08342820405960083,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000848233699798584,"max_error_over_tolerance": 0.008046787232160568,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.0324886292219162,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6608329713344574, + "samples_ms": [0.7434599101543427, 0.6608329713344574, 0.6479956209659576], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.28777867555618286, + "samples_ms": [0.31236186623573303, 0.28777867555618286, 0.2872943878173828], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.35877153277397156, + "samples_ms": [0.35877153277397156, 0.3423057496547699, 0.40040165185928345], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 2.3678094148635864, + "samples_ms": [2.812262624502182, 2.3678094148635864, 2.308174967765808], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08248910307884216, + "samples_ms": [0.10496005415916443, 0.08248910307884216, 0.08038058876991272], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.0824166238307953, + "samples_ms": [1.0685622692108154, 1.1040419340133667, 1.0824166238307953], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014126300811767578,"max_error_over_tolerance": 0.10739777982234955,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009508132934570312,"max_error_over_tolerance": 0.009040186181664467,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.055741358548402786,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.46596676111221313, + "samples_ms": [0.5198419094085693, 0.46596676111221313, 0.4613921046257019], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2003796398639679, + "samples_ms": [0.20474568009376526, 0.19324570894241333, 0.2003796398639679], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.19286200404167175, + "samples_ms": [0.20584836602210999, 0.19286200404167175, 0.18569082021713257], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.548110067844391, + "samples_ms": [4.740048199892044, 4.08400222659111, 4.548110067844391], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.10885298252105713, + "samples_ms": [0.10885298252105713, 0.09490549564361572, 0.25159865617752075], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.271624118089676, + "samples_ms": [1.271624118089676, 1.352418214082718, 1.170370727777481], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01564645767211914,"max_error_over_tolerance": 0.12006834149360657,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625476837158203,"max_error_over_tolerance": 0.08660759031772614,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6065703928470612, + "samples_ms": [0.6653815507888794, 0.6065703928470612, 0.5939602851867676], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.28814375400543213, + "samples_ms": [0.28814375400543213, 0.26183947920799255, 0.29589608311653137], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.32318755984306335, + "samples_ms": [0.34091994166374207, 0.31236931681632996, 0.32318755984306335], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.4117773175239563, + "samples_ms": [3.8426965475082397, 3.3334828913211823, 3.4117773175239563], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1622922718524933, + "samples_ms": [0.1622922718524933, 0.14113634824752808, 0.18216297030448914], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.8786080181598663, + "samples_ms": [2.4149343371391296, 1.8786080181598663, 1.6277916729450226], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 1, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.006100177764892578,"max_error_over_tolerance": 0.067171111702919,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0007508695125579834,"max_error_over_tolerance": 0.0071157063357532024,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.05316777154803276,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6371252238750458, + "samples_ms": [0.6371252238750458, 0.5845017731189728, 0.7134564220905304], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.24210289120674133, + "samples_ms": [0.2524815499782562, 0.24210289120674133, 0.23226439952850342], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2515725791454315, + "samples_ms": [0.2801157534122467, 0.2515725791454315, 0.2417340874671936], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.0545853078365326, + "samples_ms": [3.012627363204956, 3.1444616615772247, 3.0545853078365326], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1009032130241394, + "samples_ms": [0.10890886187553406, 0.1009032130241394, 0.0995248556137085], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.6249621510505676, + "samples_ms": [0.841282308101654, 0.6249621510505676, 0.4223547875881195], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007819890975952148,"max_error_over_tolerance": 0.08643733710050583,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009521245956420898,"max_error_over_tolerance": 0.009015905670821667,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014435768127441406,"max_error_over_tolerance": 0.07634487003087997,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6174556910991669, + "samples_ms": [0.528968870639801, 0.6931237876415253, 0.6174556910991669], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.215090811252594, + "samples_ms": [0.22701174020767212, 0.215090811252594, 0.21107494831085205], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.21208450198173523, + "samples_ms": [0.22354722023010254, 0.21208450198173523, 0.20722299814224243], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 2.8594695031642914, + "samples_ms": [3.271866589784622, 2.7009136974811554, 2.8594695031642914], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08808448910713196, + "samples_ms": [0.10520219802856445, 0.08808448910713196, 0.08613988757133484], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.49784407019615173, + "samples_ms": [0.7374957203865051, 0.49784407019615173, 0.3265179693698883], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078012943267822266,"max_error_over_tolerance": 0.08598456531763077,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009499192237854004,"max_error_over_tolerance": 0.009035039693117142,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015491485595703125,"max_error_over_tolerance": 0.08537446707487106,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6163567304611206, + "samples_ms": [0.6814412772655487, 0.6163567304611206, 0.5919598042964935], + "peak_allocated_bytes": 231888896, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2610310912132263, + "samples_ms": [0.26984885334968567, 0.25866180658340454, 0.2610310912132263], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.32006576657295227, + "samples_ms": [0.370122492313385, 0.32006576657295227, 0.31005963683128357], + "peak_allocated_bytes": 391910912, + "incremental_peak_bytes": 161409024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.758283495903015, + "samples_ms": [4.77876141667366, 4.7095417976379395, 4.758283495903015], + "peak_allocated_bytes": 565443584, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1256018877029419, + "samples_ms": [0.13486668467521667, 0.1256018877029419, 0.1223050057888031], + "peak_allocated_bytes": 231888896, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.401267945766449, + "samples_ms": [1.401267945766449, 1.3131499290466309, 1.4289245009422302], + "peak_allocated_bytes": 389728256, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 1, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.52587890625e-05,"max_error_over_tolerance": 0.00030462356517091393,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0004891455173492432,"max_error_over_tolerance": 0.0047654458321630955,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.03244646266102791,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5212686955928802, + "samples_ms": [0.5578920245170593, 0.5212686955928802, 0.5110912024974823], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.5053989589214325, + "samples_ms": [0.49149245023727417, 0.6247162818908691, 0.5053989589214325], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.28608739376068115, + "samples_ms": [0.37166476249694824, 0.28608739376068115, 0.26038289070129395], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 2.69177183508873, + "samples_ms": [2.622220665216446, 2.69177183508873, 2.7831755578517914], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08979439735412598, + "samples_ms": [0.09759515523910522, 0.08979439735412598, 0.08885189890861511], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.7829070091247559, + "samples_ms": [0.7398799061775208, 0.7829070091247559, 0.828668475151062], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.527508720755577e-05,"max_error_over_tolerance": 0.00030498611158691347,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000972449779510498,"max_error_over_tolerance": 0.009215136989951134,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.055741358548402786,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.620037317276001, + "samples_ms": [0.6789267063140869, 0.620037317276001, 0.599246472120285], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2673007547855377, + "samples_ms": [0.2936124801635742, 0.2673007547855377, 0.2442002296447754], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.3009811043739319, + "samples_ms": [0.3041587769985199, 0.2967715263366699, 0.3009811043739319], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 7.314201444387436, + "samples_ms": [9.714469313621521, 6.630800664424896, 7.314201444387436], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11700019240379333, + "samples_ms": [0.11700019240379333, 0.11009350419044495, 0.11860206723213196], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.337241381406784, + "samples_ms": [1.394171267747879, 1.3254545629024506, 1.337241381406784], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "cancellation", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 1.5273690223693848e-05,"max_error_over_tolerance": 0.0003049805236514658,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009714066982269287,"max_error_over_tolerance": 0.009242905303835869,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625,"max_error_over_tolerance": 0.08660212904214859,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6686709821224213, + "samples_ms": [0.7194802165031433, 0.6686709821224213, 0.6463862955570221], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.29221922159194946, + "samples_ms": [0.322144478559494, 0.29221922159194946, 0.2859048545360565], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.3761462867259979, + "samples_ms": [0.3761462867259979, 0.3552660346031189, 0.3951750695705414], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.301741719245911, + "samples_ms": [6.697036325931549, 4.301741719245911, 3.891546279191971], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11081993579864502, + "samples_ms": [0.11987611651420593, 0.11081993579864502, 0.1092180609703064], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.0448917746543884, + "samples_ms": [1.2847892940044403, 1.0448917746543884, 0.5874298512935638], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "checkpoint_fixture", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12514114379882812,"max_error_over_tolerance": 0.17978119850158691,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015633583068847656,"max_error_over_tolerance": 0.08594760298728943,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015613555908203125,"max_error_over_tolerance": 0.08570058643817902,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.46795979142189026, + "samples_ms": [1.3030916452407837, 0.46795979142189026, 0.4540979862213135], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.3589130938053131, + "samples_ms": [0.2717040479183197, 0.37369877099990845, 0.3589130938053131], + "peak_allocated_bytes": 233738240, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.41437894105911255, + "samples_ms": [0.6911270320415497, 0.41437894105911255, 0.23768097162246704], + "peak_allocated_bytes": 390488064, + "incremental_peak_bytes": 160448512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.3845027685165405, + "samples_ms": [5.283676087856293, 4.364609718322754, 4.3845027685165405], + "peak_allocated_bytes": 563594240, + "incremental_peak_bytes": 333554688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11146068572998047, + "samples_ms": [0.12151896953582764, 0.11146068572998047, 0.1089051365852356], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.294311136007309, + "samples_ms": [1.3019554316997528, 1.2869052588939667, 1.294311136007309], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + } + ], + "acceptance_complete": false, + "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", + "provenance": { + "git_commit": "32b765ec992cd1206517104ec66506881203c91c", + "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", + "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", + "python": "3.10.12", + "torch": "2.6.0+cu124", + "torch_cuda": "12.4", + "torch_hip": null, + "contract_version": "ws1-c1-v2", + "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "args": { + "rows": "1,33,129", + "families": "random,swiglu_like,cancellation", + "repeats": 3, + "seed": 420, + "output": "artifacts/h3-down/h100-fp32-cold-repeat/report.json", + "build_log": null, + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "compare_vllm": false, + "fixture": "/workspace/h3-fixtures/transformer-block0-down.pt" + }, + "environment": { + "KERNEL_ALIGN_USE_FAST_MATH": null, + "KERNEL_ALIGN_DET_GEMM_SM90": null, + "RL_KERNEL_DET_GEMM_BACKEND": null, + "CUBLAS_WORKSPACE_CONFIG": null, + "CUBLASLT_WORKSPACE_SIZE": null, + "TORCH_CUDA_ARCH_LIST": null + }, + "source_hashes": { + "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", + "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", + "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", + "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", + "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", + "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", + "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", + "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", + "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" + }, + "build_flags_verified_at_runtime": false, + "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", + "device": "NVIDIA H100 80GB HBM3", + "capability": [9, 0], + "candidate_tf32_enabled": false, + "reference_tf32_enabled": false, + "torch_baseline_bf16_reduced_precision_reduction": false, + "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" + }, + "fixture": { + "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", + "metadata": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + } + }, + "remaining_acceptance": ["maintainer review of arithmetic contract and covered matrix", "independent build/environment repeat and prior-art performance decision", "real captured SwiGLU activations"] + }, + "triton_3_4_0": { + "schema": "h3-down-qualification-v2", + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "status": "passed", + "cases": [ + { + "rows": 1, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0077440738677978516,"max_error_over_tolerance": 0.08342820405960083,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000848233699798584,"max_error_over_tolerance": 0.008046787232160568,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.0324886292219162,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.9755549728870392, + "samples_ms": [0.756826251745224, 2.429340034723282, 2.3806132376194, 1.9755549728870392, 1.5296190977096558], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.8079968392848969, + "samples_ms": [0.9192898869514465, 0.867113471031189, 0.7740408182144165, 0.8079968392848969, 0.723525881767273], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.27946382761001587, + "samples_ms": [0.3366135060787201, 0.2865418791770935, 0.2709105610847473, 0.272214412689209, 0.27946382761001587], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.154114216566086, + "samples_ms": [5.06950169801712, 5.154114216566086, 5.122512578964233, 5.297519266605377, 5.531996488571167], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08181482553482056, + "samples_ms": [0.10541826486587524, 0.08444488048553467, 0.08181482553482056, 0.0813119113445282, 0.08007138967514038], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.8751248717308044, + "samples_ms": [1.8434450030326843, 2.448052167892456, 2.0304471254348755, 1.5846826136112213, 1.8751248717308044], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "442ea5185d0fd78f99a7d9e7bfc6024673a950fa98f4315816aa22c4a38c5053","cubin_sha256": "0be9cc59dade284d97c8c7831a3e463e5e96b8c3dcb63cab3962c9ea3141163d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "71a486a14ced26068c5e89b957e640da2b1ce1472c1bab9830d521e55fd5de55","cubin_sha256": "3de63ceb61e4bcaf96a4ce172f63ce70126fb67774c1c6a56c5dd22a489ccfda","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06","cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "vllm_comparison": { + "status": "passed_probe", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0077440738677978516,"max_error_over_tolerance": 0.08342820405960083,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000848233699798584,"max_error_over_tolerance": 0.008046787232160568,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.0324886292219162,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton"} + ], + "benchmarks": { + "forward": { + "median_ms": 0.35585835576057434, + "samples_ms": [0.39868801832199097, 0.35585835576057434, 0.36633387207984924, 0.33082813024520874, 0.32956525683403015], + "peak_allocated_bytes": 838020608, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dx": { + "median_ms": 0.3727339208126068, + "samples_ms": [0.42771175503730774, 0.38250163197517395, 0.3697015345096588, 0.3727339208126068, 0.3604479134082794], + "peak_allocated_bytes": 838060032, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dw": { + "median_ms": 0.23178011178970337, + "samples_ms": [0.28607621788978577, 0.25910884141921997, 0.22685527801513672, 0.23178011178970337, 0.2170279622077942], + "peak_allocated_bytes": 992207872, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", + "triton_version": "3.4.0", + "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", + "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", + "configs": [ + { + "shape": { + "M": 1, + "N": 5376, + "K": 14336 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 1, + "N": 14336, + "K": 5376 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 5376, + "N": 14336, + "K": 1 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + } + ], + "qualification": "partial_prior_art_probe_only" + }, + "passed": true + }, + { + "rows": 129, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01564645767211914,"max_error_over_tolerance": 0.12006834149360657,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625476837158203,"max_error_over_tolerance": 0.08660759031772614,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5118362605571747, + "samples_ms": [0.7387325167655945, 1.6151517629623413, 0.48866868019104004, 0.5118362605571747, 0.48043206334114075], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.20536035299301147, + "samples_ms": [0.22369623184204102, 0.2122148871421814, 0.20536035299301147, 0.2004280686378479, 0.20079687237739563], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.23826584219932556, + "samples_ms": [0.2539418637752533, 0.23826584219932556, 0.23221969604492188, 0.2310052514076233, 0.24832412600517273], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.4094473123550415, + "samples_ms": [4.275541752576828, 4.517361521720886, 4.4094473123550415, 4.568081349134445, 4.234585911035538], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.12133270502090454, + "samples_ms": [0.1527555286884308, 0.12133270502090454, 0.12947991490364075, 0.12053549289703369, 0.11564046144485474], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.3994351029396057, + "samples_ms": [1.2967772781848907, 1.314643770456314, 1.3994351029396057, 1.4288835227489471, 1.4533214271068573], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "b8fc72a5623290859b63cc1099242078498d7aa6da0a319c35c9e6026617f553","cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "999ed0c8d411455da93d5dd63f766ec664ac762855dd057d45edad2325e0801d","cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06","cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "vllm_comparison": { + "status": "passed_probe", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01564645767211914,"max_error_over_tolerance": 0.12006834149360657,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009601116180419922,"max_error_over_tolerance": 0.009125219658017159,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015625476837158203,"max_error_over_tolerance": 0.08660759031772614,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton"} + ], + "benchmarks": { + "forward": { + "median_ms": 0.36650896072387695, + "samples_ms": [0.4074461758136749, 0.3713257610797882, 0.36650896072387695, 0.36140158772468567, 0.3532879054546356], + "peak_allocated_bytes": 865554944, + "incremental_peak_bytes": 2311680, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dx": { + "median_ms": 0.4915297031402588, + "samples_ms": [0.5265325307846069, 0.4915297031402588, 0.4705600440502167, 0.4768744111061096, 0.6239041686058044], + "peak_allocated_bytes": 869264384, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dw": { + "median_ms": 0.3756880760192871, + "samples_ms": [0.4069618880748749, 0.3756880760192871, 0.36858394742012024, 0.3760494291782379, 0.3723315894603729], + "peak_allocated_bytes": 1021111296, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", + "triton_version": "3.4.0", + "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", + "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", + "configs": [ + { + "shape": { + "M": 129, + "N": 5376, + "K": 14336 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 129, + "N": 14336, + "K": 5376 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 5376, + "N": 14336, + "K": 129 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + } + ], + "qualification": "partial_prior_art_probe_only" + }, + "passed": true + }, + { + "rows": 8192, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015656471252441406,"max_error_over_tolerance": 0.11954245716333389,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000977635383605957,"max_error_over_tolerance": 0.009301135316491127,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12531661987304688,"max_error_over_tolerance": 0.1686829924583435,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 7.39080086350441, + "samples_ms": [7.39080086350441, 7.132500410079956, 6.938610225915909, 7.768817245960236, 8.397072553634644], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 4.552166908979416, + "samples_ms": [4.7802068293094635, 4.4778212904930115, 4.534110426902771, 4.614364355802536, 4.552166908979416], + "peak_allocated_bytes": 1014235136, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 4.351019859313965, + "samples_ms": [4.253793507814407, 4.231806844472885, 4.351019859313965, 4.374224692583084, 4.366464912891388], + "peak_allocated_bytes": 933494784, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 19.10606399178505, + "samples_ms": [22.746942937374115, 19.242648035287857, 18.63836869597435, 18.933270126581192, 19.10606399178505], + "peak_allocated_bytes": 2291867648, + "incremental_peak_bytes": 1512513536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 1.7073601484298706, + "samples_ms": [1.7186403274536133, 1.7073601484298706, 1.7057321965694427, 1.7052367329597473, 1.7091743648052216], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 5.997553467750549, + "samples_ms": [5.202878266572952, 5.3045786917209625, 5.997553467750549, 9.920638054609299, 8.24371725320816], + "peak_allocated_bytes": 1256456192, + "incremental_peak_bytes": 477102080, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "96b3410b50e0ebcc24825af8569468d59d46882af5be72b7bfd8ee36bad4f2c2","cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105","compiled_num_warps": 4,"grid": [128,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "bb8d38510563eb1678e3e76afe4e649f1447d27e9734ccd6ef2eb116502c5d58","cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64","compiled_num_warps": 4,"grid": [128,224]},"canonical_active_rows": 8192,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06","cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8", + "dtype": "torch.bfloat16", + "shape": [8192, 5376] + }, + "dx": { + "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66", + "dtype": "torch.bfloat16", + "shape": [8192, 14336] + }, + "dw": { + "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "vllm_comparison": { + "status": "passed_probe", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015656471252441406,"max_error_over_tolerance": 0.11954245716333389,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000977635383605957,"max_error_over_tolerance": 0.009301135316491127,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12531661987304688,"max_error_over_tolerance": 0.1686829924583435,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton"} + ], + "benchmarks": { + "forward": { + "median_ms": 2.499658614397049, + "samples_ms": [2.5562308728694916, 2.5168806314468384, 2.499658614397049, 2.48013436794281, 2.489827573299408], + "peak_allocated_bytes": 2540830720, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dx": { + "median_ms": 5.712740123271942, + "samples_ms": [5.7429783046245575, 5.476012825965881, 5.496017634868622, 5.7668983936309814, 5.712740123271942], + "peak_allocated_bytes": 2775722496, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dw": { + "median_ms": 10.247066617012024, + "samples_ms": [10.233435779809952, 10.858424007892609, 10.247066617012024, 10.259285569190979, 10.230351239442825], + "peak_allocated_bytes": 2841800704, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", + "triton_version": "3.4.0", + "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", + "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", + "configs": [ + { + "shape": { + "M": 8192, + "N": 5376, + "K": 14336 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 8192, + "N": 14336, + "K": 5376 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 5376, + "N": 14336, + "K": 8192 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + } + ], + "qualification": "partial_prior_art_probe_only" + }, + "passed": true + }, + { + "rows": 32768, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01569652557373047,"max_error_over_tolerance": 0.12013405561447144,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781122207641602,"max_error_over_tolerance": 0.009311916306614876,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.2525482177734375,"max_error_over_tolerance": 0.18232759833335876,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 41.59632697701454, + "samples_ms": [41.687577962875366, 41.54922440648079, 41.800860315561295, 41.590429842472076, 41.59632697701454], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 31.687993556261063, + "samples_ms": [29.00845929980278, 24.406280368566513, 33.68991240859032, 31.687993556261063, 31.89874067902565], + "peak_allocated_bytes": 3393191936, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 20.97996324300766, + "samples_ms": [19.690733402967453, 21.237235516309738, 20.029544830322266, 22.116035223007202, 20.97996324300766], + "peak_allocated_bytes": 2607808512, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 99.08927977085114, + "samples_ms": [97.93517366051674, 99.08927977085114, 107.6027862727642, 96.2594524025917, 99.96020048856735], + "peak_allocated_bytes": 7578877952, + "incremental_peak_bytes": 5125210112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 6.319291889667511, + "samples_ms": [6.33331760764122, 6.319291889667511, 6.296105682849884, 6.304707378149033, 7.043942809104919], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 20.878225564956665, + "samples_ms": [25.30929446220398, 21.993432193994522, 20.878225564956665, 20.874205976724625, 20.87520807981491], + "peak_allocated_bytes": 3899654144, + "incremental_peak_bytes": 1445986304, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "96b3410b50e0ebcc24825af8569468d59d46882af5be72b7bfd8ee36bad4f2c2","cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105","compiled_num_warps": 4,"grid": [512,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "bb8d38510563eb1678e3e76afe4e649f1447d27e9734ccd6ef2eb116502c5d58","cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64","compiled_num_warps": 4,"grid": [512,224]},"canonical_active_rows": 32768,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06","cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949", + "dtype": "torch.bfloat16", + "shape": [32768, 5376] + }, + "dx": { + "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515", + "dtype": "torch.bfloat16", + "shape": [32768, 14336] + }, + "dw": { + "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "vllm_comparison": { + "status": "passed_probe", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01569652557373047,"max_error_over_tolerance": 0.12013405561447144,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009781122207641602,"max_error_over_tolerance": 0.009311916306614876,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.2525482177734375,"max_error_over_tolerance": 0.18232759833335876,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton"} + ], + "benchmarks": { + "forward": { + "median_ms": 9.66041162610054, + "samples_ms": [9.503252804279327, 9.553678333759308, 10.058004409074783, 9.66041162610054, 9.696289896965027], + "peak_allocated_bytes": 7649886208, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dx": { + "median_ms": 21.029628813266754, + "samples_ms": [21.116141229867935, 21.001089364290237, 20.985208451747894, 21.029628813266754, 21.04152739048004], + "peak_allocated_bytes": 8589421056, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dw": { + "median_ms": 40.02102091908455, + "samples_ms": [40.06137698888779, 40.02102091908455, 40.013208985328674, 39.991073310375214, 40.18115624785423], + "peak_allocated_bytes": 8391258112, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", + "triton_version": "3.4.0", + "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", + "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", + "configs": [ + { + "shape": { + "M": 32768, + "N": 5376, + "K": 14336 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 32768, + "N": 14336, + "K": 5376 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 5376, + "N": 14336, + "K": 32768 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + } + ], + "qualification": "partial_prior_art_probe_only" + }, + "passed": true + }, + { + "rows": 129, + "input_family": "checkpoint_fixture", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12514114379882812,"max_error_over_tolerance": 0.17978119850158691,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015633583068847656,"max_error_over_tolerance": 0.08594760298728943,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015613555908203125,"max_error_over_tolerance": 0.08570058643817902,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4480406641960144, + "samples_ms": [0.5202442407608032, 0.46155601739883423, 0.4480406641960144, 0.44644251465797424, 0.4439912736415863], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19307062029838562, + "samples_ms": [0.20404160022735596, 0.20174309611320496, 0.18886849284172058, 0.19307062029838562, 0.1902841031551361], + "peak_allocated_bytes": 233738240, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.22620707750320435, + "samples_ms": [0.2653747797012329, 0.22744014859199524, 0.22080540657043457, 0.21801888942718506, 0.22620707750320435], + "peak_allocated_bytes": 390488064, + "incremental_peak_bytes": 160448512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.0629821121692657, + "samples_ms": [2.863481640815735, 3.0629821121692657, 3.391604870557785, 3.008224070072174, 3.2156631350517273], + "peak_allocated_bytes": 563594240, + "incremental_peak_bytes": 333554688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1094229519367218, + "samples_ms": [0.12498721480369568, 0.1124851405620575, 0.1094229519367218, 0.10900571942329407, 0.1081712543964386], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.41426345705986023, + "samples_ms": [0.5338490009307861, 0.3857165575027466, 0.5918629467487335, 0.41426345705986023, 0.36514177918434143], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "b8fc72a5623290859b63cc1099242078498d7aa6da0a319c35c9e6026617f553","cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "999ed0c8d411455da93d5dd63f766ec664ac762855dd057d45edad2325e0801d","cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.4.0","compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06","cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "vllm_comparison": { + "status": "passed_probe", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12514114379882812,"max_error_over_tolerance": 0.17978119850158691,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015633583068847656,"max_error_over_tolerance": 0.08594760298728943,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015613555908203125,"max_error_over_tolerance": 0.08570058643817902,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton"} + ], + "benchmarks": { + "forward": { + "median_ms": 0.31507760286331177, + "samples_ms": [0.33548474311828613, 0.32701343297958374, 0.31507760286331177, 0.3068968653678894, 0.30349940061569214], + "peak_allocated_bytes": 865554944, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dx": { + "median_ms": 0.41652098298072815, + "samples_ms": [0.4444718360900879, 0.41950494050979614, 0.41652098298072815, 0.3989376127719879, 0.397317111492157], + "peak_allocated_bytes": 869264384, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "dw": { + "median_ms": 0.32254308462142944, + "samples_ms": [0.5048699676990509, 0.47458335757255554, 0.32254308462142944, 0.31186267733573914, 0.30764564871788025], + "peak_allocated_bytes": 1022035968, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", + "triton_version": "3.4.0", + "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", + "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", + "configs": [ + { + "shape": { + "M": 129, + "N": 5376, + "K": 14336 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 129, + "N": 14336, + "K": 5376 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + }, + { + "shape": { + "M": 5376, + "N": 14336, + "K": 129 + }, + "dtype": "bfloat16", + "config": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8 + }, + "selection": "default", + "is_tuned": false, + "device_capability": [9, 0], + "arch_family": "hopper" + } + ], + "qualification": "partial_prior_art_probe_only" + }, + "passed": true + } + ], + "acceptance_complete": false, + "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", + "provenance": { + "git_commit": "32b765ec992cd1206517104ec66506881203c91c", + "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", + "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", + "python": "3.10.12", + "torch": "2.6.0+cu124", + "torch_cuda": "12.4", + "torch_hip": null, + "contract_version": "ws1-c1-v2", + "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "args": { + "rows": "1,129,8192,32768", + "families": "random", + "repeats": 5, + "seed": 420, + "output": "artifacts/h3-down/h100-triton34-comparison/report.json", + "build_log": null, + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "compare_vllm": true, + "fixture": "/workspace/h3-fixtures/transformer-block0-down.pt" + }, + "environment": { + "KERNEL_ALIGN_USE_FAST_MATH": null, + "KERNEL_ALIGN_DET_GEMM_SM90": null, + "RL_KERNEL_DET_GEMM_BACKEND": null, + "CUBLAS_WORKSPACE_CONFIG": null, + "CUBLASLT_WORKSPACE_SIZE": null, + "TORCH_CUDA_ARCH_LIST": null + }, + "source_hashes": { + "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", + "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", + "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", + "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", + "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", + "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", + "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", + "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", + "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" + }, + "build_flags_verified_at_runtime": false, + "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", + "device": "NVIDIA H100 80GB HBM3", + "capability": [9, 0], + "candidate_tf32_enabled": false, + "reference_tf32_enabled": false, + "torch_baseline_bf16_reduced_precision_reduction": false, + "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" + }, + "fixture": { + "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", + "metadata": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + } + }, + "remaining_acceptance": ["maintainer review of arithmetic contract and covered matrix", "independent build/environment repeat and prior-art performance decision", "real captured SwiGLU activations"] + }, + "cold_cache_output_comparison": { + "status": "passed", + "scope": "Same H100 and PyTorch profile; separate process and fresh Triton 3.2.0 cache; not another GPU or runtime environment", + "checks": [ + {"input_family": "random","rows": 1,"tensor": "y","passed": true,"sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386"}, + {"input_family": "random","rows": 1,"tensor": "dx","passed": true,"sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f"}, + {"input_family": "random","rows": 1,"tensor": "dw","passed": true,"sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9"}, + {"input_family": "random","rows": 33,"tensor": "y","passed": true,"sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869"}, + {"input_family": "random","rows": 33,"tensor": "dx","passed": true,"sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee"}, + {"input_family": "random","rows": 33,"tensor": "dw","passed": true,"sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08"}, + {"input_family": "random","rows": 129,"tensor": "y","passed": true,"sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7"}, + {"input_family": "random","rows": 129,"tensor": "dx","passed": true,"sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d"}, + {"input_family": "random","rows": 129,"tensor": "dw","passed": true,"sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078"}, + {"input_family": "swiglu_like","rows": 1,"tensor": "y","passed": true,"sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb"}, + {"input_family": "swiglu_like","rows": 1,"tensor": "dx","passed": true,"sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43"}, + {"input_family": "swiglu_like","rows": 1,"tensor": "dw","passed": true,"sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda"}, + {"input_family": "swiglu_like","rows": 33,"tensor": "y","passed": true,"sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e"}, + {"input_family": "swiglu_like","rows": 33,"tensor": "dx","passed": true,"sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60"}, + {"input_family": "swiglu_like","rows": 33,"tensor": "dw","passed": true,"sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f"}, + {"input_family": "swiglu_like","rows": 129,"tensor": "y","passed": true,"sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236"}, + {"input_family": "swiglu_like","rows": 129,"tensor": "dx","passed": true,"sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8"}, + {"input_family": "swiglu_like","rows": 129,"tensor": "dw","passed": true,"sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f"}, + {"input_family": "cancellation","rows": 1,"tensor": "y","passed": true,"sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421"}, + {"input_family": "cancellation","rows": 1,"tensor": "dx","passed": true,"sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730"}, + {"input_family": "cancellation","rows": 1,"tensor": "dw","passed": true,"sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3"}, + {"input_family": "cancellation","rows": 33,"tensor": "y","passed": true,"sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b"}, + {"input_family": "cancellation","rows": 33,"tensor": "dx","passed": true,"sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3"}, + {"input_family": "cancellation","rows": 33,"tensor": "dw","passed": true,"sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18"}, + {"input_family": "cancellation","rows": 129,"tensor": "y","passed": true,"sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9"}, + {"input_family": "cancellation","rows": 129,"tensor": "dx","passed": true,"sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930"}, + {"input_family": "cancellation","rows": 129,"tensor": "dw","passed": true,"sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "y","passed": true,"sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "dx","passed": true,"sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "dw","passed": true,"sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41"} + ] + }, + "compiler_version_output_comparison": { + "status": "passed", + "scope": "Same H100/PyTorch; Triton 3.2.0 versus separately pinned 3.4.0 using independent JIT caches; selected cases only", + "checks": [ + {"input_family": "random","rows": 1,"tensor": "y","passed": true,"sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386"}, + {"input_family": "random","rows": 1,"tensor": "dx","passed": true,"sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f"}, + {"input_family": "random","rows": 1,"tensor": "dw","passed": true,"sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9"}, + {"input_family": "random","rows": 129,"tensor": "y","passed": true,"sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7"}, + {"input_family": "random","rows": 129,"tensor": "dx","passed": true,"sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d"}, + {"input_family": "random","rows": 129,"tensor": "dw","passed": true,"sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078"}, + {"input_family": "random","rows": 8192,"tensor": "y","passed": true,"sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8"}, + {"input_family": "random","rows": 8192,"tensor": "dx","passed": true,"sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66"}, + {"input_family": "random","rows": 8192,"tensor": "dw","passed": true,"sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328"}, + {"input_family": "random","rows": 32768,"tensor": "y","passed": true,"sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949"}, + {"input_family": "random","rows": 32768,"tensor": "dx","passed": true,"sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515"}, + {"input_family": "random","rows": 32768,"tensor": "dw","passed": true,"sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "y","passed": true,"sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "dx","passed": true,"sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84"}, + {"input_family": "checkpoint_fixture","rows": 129,"tensor": "dw","passed": true,"sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41"} + ] + } +} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json b/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json new file mode 100644 index 000000000..6438072a4 --- /dev/null +++ b/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json @@ -0,0 +1,672 @@ +{ + "schema": "h3-down-evidence-summary-v1", + "readiness": "ready_for_operator_review", + "production_qualified": false, + "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", + "full_report_header": { + "schema": "h3-down-qualification-v2", + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "status": "passed", + "acceptance_complete": false, + "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", + "provenance": { + "git_commit": "32b765ec992cd1206517104ec66506881203c91c", + "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", + "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", + "python": "3.10.12", + "torch": "2.6.0+cu124", + "torch_cuda": "12.4", + "torch_hip": null, + "contract_version": "ws1-c1-v2", + "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "args": { + "rows": "1,31,32,33,127,128,129,256,513,1024,2048,4096,8192,16385,24577,32768", + "families": "random,swiglu_like,cancellation", + "repeats": 5, + "seed": 420, + "output": "artifacts/h3-down/h100-fp32-full/report.json", + "build_log": null, + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "compare_vllm": false, + "fixture": "/workspace/h3-fixtures/transformer-block0-down.pt" + }, + "environment": { + "KERNEL_ALIGN_USE_FAST_MATH": null, + "KERNEL_ALIGN_DET_GEMM_SM90": null, + "RL_KERNEL_DET_GEMM_BACKEND": null, + "CUBLAS_WORKSPACE_CONFIG": null, + "CUBLASLT_WORKSPACE_SIZE": null, + "TORCH_CUDA_ARCH_LIST": null + }, + "source_hashes": { + "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", + "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", + "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", + "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", + "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", + "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", + "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", + "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", + "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" + }, + "build_flags_verified_at_runtime": false, + "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", + "device": "NVIDIA H100 80GB HBM3", + "capability": [9, 0], + "candidate_tf32_enabled": false, + "reference_tf32_enabled": false, + "torch_baseline_bf16_reduced_precision_reduction": false, + "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" + }, + "fixture": { + "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", + "metadata": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + } + }, + "remaining_acceptance": ["maintainer review of arithmetic contract and covered matrix", "independent build/environment repeat and prior-art performance decision", "real captured SwiGLU activations"] + }, + "fixture_case_indexes": [48], + "fixture_cases": [ + { + "rows": 129, + "input_family": "checkpoint_fixture", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12514114379882812,"max_error_over_tolerance": 0.17978119850158691,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015633583068847656,"max_error_over_tolerance": 0.08594760298728943,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015613555908203125,"max_error_over_tolerance": 0.08570058643817902,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.498991459608078, + "samples_ms": [0.5232095718383789, 0.498991459608078, 0.5095936357975006, 0.4890933632850647, 0.48211216926574707], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.20795688033103943, + "samples_ms": [0.22409483790397644, 0.2106502652168274, 0.2070702612400055, 0.20523741841316223, 0.20795688033103943], + "peak_allocated_bytes": 233738240, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2506338059902191, + "samples_ms": [0.27573853731155396, 0.2506338059902191, 0.25071948766708374, 0.2502724528312683, 0.24274364113807678], + "peak_allocated_bytes": 390488064, + "incremental_peak_bytes": 160448512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.032248795032501, + "samples_ms": [5.024779587984085, 4.45270910859108, 5.032248795032501, 5.378324538469315, 8.565746247768402], + "peak_allocated_bytes": 563594240, + "incremental_peak_bytes": 333554688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11239573359489441, + "samples_ms": [0.12648478150367737, 0.11574849486351013, 0.11239573359489441, 0.11062994599342346, 0.11038035154342651], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.329515129327774, + "samples_ms": [1.329515129327774, 1.3584904372692108, 1.4058314263820648, 0.9705126285552979, 0.9936131536960602], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + } + ], + "full_matrix_summary": { + "cases": 49, + "checks": 1141, + "passed_checks": 1141, + "max_error_over_tolerance": 0.18232759833335876 + }, + "fixture_provenance": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + }, + "gpu_tests": { + "status": "passed", + "passed": 64, + "command": "python -m pytest -q tests/test_h3_ffn_down.py tests/test_h3_down_tooling.py tests/test_h3_down_fixture.py tests/test_tolerance_contract.py", + "original_stdout": "................................................................ [100%]\n64 passed in 16.70s\n", + "original_stdout_sha256": "89b85664fcf2fc6e0dc519e6656d0dddfe00c006ed98731ff2d9f126aba52c86" + }, + "rejected_native_candidate": { + "schema": "h3-down-qualification-v1", + "backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", + "status": "failed", + "cases": [ + { + "rows": 1, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.024131298065185547,"max_error_over_tolerance": 0.3214894235134125,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.001803085207939148,"max_error_over_tolerance": 0.017243297770619392,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00390625,"max_error_over_tolerance": 0.0324886292219162,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "native_forward": { + "median_ms": 2.529580146074295, + "samples_ms": [2.5643259286880493, 2.473689615726471, 2.529580146074295], + "peak_allocated_bytes": 226376192, + "incremental_peak_bytes": 5057024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dx": { + "median_ms": 2.3520365357398987, + "samples_ms": [2.546001225709915, 2.3520365357398987, 2.1655037999153137], + "peak_allocated_bytes": 380534784, + "incremental_peak_bytes": 159215616, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dw_canonical_order": { + "median_ms": 1.4085769653320312, + "samples_ms": [1.4320090413093567, 1.4085769653320312, 1.4054067432880402], + "peak_allocated_bytes": 377982976, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_forward_backward": { + "median_ms": 13.572029769420624, + "samples_ms": [12.907318770885468, 13.572029769420624, 13.592474162578583], + "peak_allocated_bytes": 534715392, + "incremental_peak_bytes": 313396224, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11911988258361816, + "samples_ms": [0.16237422823905945, 0.11911988258361816, 0.08111074566841125], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.7563531398773193, + "samples_ms": [0.484466552734375, 0.7563531398773193, 0.7724203169345856], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","fallback": false,"arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes","tile": [128,64,32],"split_k": false,"weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","canonical_active_rows": 1} + ], + "passed": true + }, + { + "rows": 33, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.03141450881958008,"max_error_over_tolerance": 0.4475856125354767,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.002236664295196533,"max_error_over_tolerance": 0.02126086875796318,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.01641559600830078,"max_error_over_tolerance": 0.11125296354293823,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "native_forward": { + "median_ms": 2.881385385990143, + "samples_ms": [3.069184720516205, 2.881385385990143, 2.5128573179244995], + "peak_allocated_bytes": 228899840, + "incremental_peak_bytes": 5401088, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dx": { + "median_ms": 2.8548426926136017, + "samples_ms": [2.9342174530029297, 2.8548426926136017, 2.647843211889267], + "peak_allocated_bytes": 383631872, + "incremental_peak_bytes": 160133120, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dw_canonical_order": { + "median_ms": 1.6956888139247894, + "samples_ms": [1.6956888139247894, 1.7749592661857605, 1.5147589147090912], + "peak_allocated_bytes": 382685696, + "incremental_peak_bytes": 159186944, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_forward_backward": { + "median_ms": 11.53586059808731, + "samples_ms": [11.53586059808731, 10.692402720451355, 14.232128858566284], + "peak_allocated_bytes": 541679616, + "incremental_peak_bytes": 318180864, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.10365620255470276, + "samples_ms": [0.11552125215530396, 0.10365620255470276, 0.1003853976726532], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.3978853821754456, + "samples_ms": [1.3978853821754456, 1.4610998332500458, 1.0400116443634033], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","fallback": false,"arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes","tile": [128,64,32],"split_k": false,"weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","canonical_active_rows": 33} + ], + "passed": true + }, + { + "rows": 129, + "input_family": "random", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.036692142486572266,"max_error_over_tolerance": 0.519758403301239,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.002099767327308655,"max_error_over_tolerance": 0.02042568475008011,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.040273189544677734,"max_error_over_tolerance": 0.2996149957180023,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "native_forward": { + "median_ms": 3.5951845347881317, + "samples_ms": [3.5951845347881317, 3.9503835141658783, 3.1119659543037415], + "peak_allocated_bytes": 241597952, + "incremental_peak_bytes": 11558400, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dx": { + "median_ms": 2.6809796690940857, + "samples_ms": [2.8591640293598175, 2.6809796690940857, 2.6392340660095215], + "peak_allocated_bytes": 398491136, + "incremental_peak_bytes": 168451584, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dw_canonical_order": { + "median_ms": 2.1124370396137238, + "samples_ms": [1.8919706344604492, 3.0186958611011505, 2.1124370396137238], + "peak_allocated_bytes": 398014464, + "incremental_peak_bytes": 167974912, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_forward_backward": { + "median_ms": 11.893197894096375, + "samples_ms": [10.98896935582161, 11.893197894096375, 12.51029223203659], + "peak_allocated_bytes": 571289088, + "incremental_peak_bytes": 341249536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1161135733127594, + "samples_ms": [0.1265592873096466, 0.1161135733127594, 0.11379271745681763], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.4508739113807678, + "samples_ms": [1.4508739113807678, 2.024903893470764, 1.4123953878879547], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","fallback": false,"arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes","tile": [128,64,32],"split_k": false,"weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","canonical_active_rows": 129} + ], + "passed": true + }, + { + "rows": 129, + "input_family": "checkpoint_fixture", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": false,"max_abs_error": 0.40984344482421875,"max_error_over_tolerance": 5.004410743713379,"failing_elements": 23353,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.042763710021972656,"max_error_over_tolerance": 0.37028035521507263,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.036380767822265625,"max_error_over_tolerance": 0.20060789585113525,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "native_forward": { + "median_ms": 4.083797335624695, + "samples_ms": [3.0049867928028107, 6.327677518129349, 4.083797335624695], + "peak_allocated_bytes": 241519104, + "incremental_peak_bytes": 11479552, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dx": { + "median_ms": 18.319733440876007, + "samples_ms": [18.319733440876007, 14.052655547857285, 21.322384476661682], + "peak_allocated_bytes": 397971456, + "incremental_peak_bytes": 167931904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_dw_canonical_order": { + "median_ms": 3.0141323804855347, + "samples_ms": [3.469545394182205, 2.9487088322639465, 3.0141323804855347], + "peak_allocated_bytes": 396795904, + "incremental_peak_bytes": 166756352, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "native_forward_backward": { + "median_ms": 11.083446443080902, + "samples_ms": [11.083446443080902, 8.68365541100502, 13.705644756555557], + "peak_allocated_bytes": 569902080, + "incremental_peak_bytes": 339862528, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11894479393959045, + "samples_ms": [0.12494996190071106, 0.11894479393959045, 0.11604651808738708], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.3583170473575592, + "samples_ms": [0.3861375153064728, 0.3583170473575592, 0.35572052001953125], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1","fallback": false,"arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes","tile": [128,64,32],"split_k": false,"weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","canonical_active_rows": 129} + ], + "passed": false + } + ], + "acceptance_complete": false, + "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", + "provenance": { + "git_commit": "32b765ec992cd1206517104ec66506881203c91c", + "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py\n?? scripts/prepare_h3_down_fixture.py", + "git_diff_sha256": "dcbbf6328bc4324e48a0753cba99e14b0618fc673080016c575f8dd5bce2cc0e", + "python": "3.10.12", + "torch": "2.6.0+cu124", + "torch_cuda": "12.4", + "torch_hip": null, + "contract_version": "ws1-c1-v2", + "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "args": { + "rows": "1,33,129", + "families": "random", + "repeats": 3, + "seed": 420, + "output": "artifacts/h3-down/h100-smoke/report.json", + "build_log": "artifacts/h3-down/h100-smoke/build.log", + "compare_vllm": false, + "fixture": "/workspace/h3-fixtures/transformer-block0-down.pt" + }, + "environment": { + "KERNEL_ALIGN_USE_FAST_MATH": "0", + "KERNEL_ALIGN_DET_GEMM_SM90": "1", + "RL_KERNEL_DET_GEMM_BACKEND": "sm90", + "CUBLAS_WORKSPACE_CONFIG": null, + "CUBLASLT_WORKSPACE_SIZE": null, + "TORCH_CUDA_ARCH_LIST": null + }, + "source_hashes": { + "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", + "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", + "rl_engine/kernels/ops/h3_ffn_down.py": "57916c658f451daf04380daff6404ad0924429b58df54566aae9aad575d1dec9", + "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", + "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", + "rl_engine/kernels/registry.py": "4680bf38f046487073945e0bcfb73e936f8c1183a4f5dff65d1699740c4c07e2", + "scripts/validate_h3_ffn_down.py": "6509c17e44ed1e8f5d5cabf145af4e6c59bfde47b388968f9ad26b9e797e2ec1" + }, + "build_flags_verified_at_runtime": false, + "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", + "build_log_sha256": "ce2722b632e6a0903f795d44d293c8c6c0da8a3835388da7db0d27e25eb2ef34", + "device": "NVIDIA H100 80GB HBM3", + "capability": [9, 0], + "candidate_tf32_enabled": false, + "reference_tf32_enabled": false, + "torch_baseline_bf16_reduced_precision_reduction": false, + "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" + }, + "fixture": { + "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", + "metadata": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + } + }, + "remaining_acceptance": ["maintainer review of arithmetic contract and covered matrix", "independent build/environment repeat and prior-art performance decision", "real captured SwiGLU activations"] + }, + "vllm_triton_3_2_error_log": "WARNING 10-10 14:37:52 [RL-Kernel]: Core binary extension (_C) unavailable: cannot import name '_C' from 'rl_engine' (/workspace/RL-Kernel/rl_engine/__init__.py). Falling back to native code.\n1 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n33 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n129 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n", + "limitations": ["Real checkpoint weights were paired with synthetic activations, not captured H3 activations.", "One SM90 GPU, eager WS1 only; no whole-model or TP qualification.", "The default fixed-tile adapter is slower than the ordinary PyTorch BF16 comparator.", "The vLLM probe does not qualify canonical autograd or all dW invariance transforms.", "Strict production resolution remains gated by the test-only descriptor."], + "post_measurement_formatting": { + "gpu_rerun_after_formatting": false, + "validation": "Black AST-equivalence check; identical non-import AST and import bindings; local regression rerun", + "files": [ + { + "path": "rl_engine/kernels/ops/h3_ffn_down.py", + "measured_source_sha256": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", + "pr_source_sha256": "4586fd466e8d3c9c13fc9906eca4788ca1d2833d2156f81086a0fe3e42b573b9", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "rl_engine/kernels/ops/h3_down_fp32_backend.py", + "measured_source_sha256": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", + "pr_source_sha256": "4fb231d6548f1ec084dd19edafba76fd4184d6050cc8d182a90c697341ce8258", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "rl_engine/kernels/registry.py", + "measured_source_sha256": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", + "pr_source_sha256": "5e7cb402a3d94c303b5178f3591cfc11c3ad7fc68610262fb1cf4ea0027e7987", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "scripts/validate_h3_ffn_down.py", + "measured_source_sha256": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772", + "pr_source_sha256": "4a16d48437ba330a24ba463bbc35f294d2e2952167472dbc72f2913f9ff75e65", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "scripts/prepare_h3_down_fixture.py", + "measured_source_sha256": "73cfaa8e7fda03bf7ce3fb4e9928d9c01949d843ee4f05407df71e9163bc9778", + "pr_source_sha256": "2accd665d2bfcc49246f9de3ea59482eeac8b0d89314462d11edd1d6582e14b7", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "tests/test_h3_ffn_down.py", + "measured_source_sha256": "bdbb5a86948f1222750508c70fa641b971eeed2dd819de1a222f656b6e2e50b0", + "pr_source_sha256": "eead795bc0dc62a760d3f09b5fbfdf9b0fb16de2bfb415cb983df01d25732308", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + }, + { + "path": "tests/test_h3_down_tooling.py", + "measured_source_sha256": "c6e17f6da1289db2bf26ef83a3a6d63c92b78d95bb9b10082d30a5bcf8f61eca", + "pr_source_sha256": "84a9ca7664edf12c8c3397f3e3383a618e8b38bb60c498512f9fbbce1247d11d", + "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", + "non_import_ast_equal": true, + "import_bindings_equal": true + } + ] + }, + "local_pr_validation": { + "status": "passed", + "passed": 85, + "skipped": 3, + "command": "python -m pytest -q tests/test_h3_ffn_down.py tests/test_h3_down_tooling.py tests/test_h3_down_fixture.py tests/test_tolerance_contract.py tests/test_kernel_registry.py tests/test_cross_config_contract.py tests/test_cross_config_runtime.py", + "stdout": "....................sss................................................. [ 81%]\n................ [100%]\n85 passed, 3 skipped in 7.10s\n", + "checks": ["Black 24.4.2, line length 100", "isort 5.13.2, Black profile, line length 100", "flake8 7.0.0, max line length 100, E203/E704 ignored", "Ruff", "bash -n scripts/run_h3_down_h100.sh", "git diff --check"], + "scope": "Local CPU-safe regression on final formatted sources; three full-width SM90 tests skipped. The earlier 64-test H100 run is preserved separately." + }, + "evidence_files": [ + { + "file": "h100-cancellation.json", + "bytes": 244600, + "sha256": "608483ed60de33c388e4bb7bc090fac71619a0d31deceefdac12ea099a6fcb09" + }, + { + "file": "h100-random.json", + "bytes": 244429, + "sha256": "97652ca2c83ba42e821a98fbc2810000152f22af2dffb826b433ce6eabcad1b0" + }, + { + "file": "h100-replays.json", + "bytes": 272824, + "sha256": "e47d4173b52ff3b2d68b314227710e91896c9753b2c485942bb4c585cf2cc89f" + }, + { + "file": "h100-swiglu-like.json", + "bytes": 244560, + "sha256": "a8f615dc2ac313cab0c872da84098e479dab69b2ad74bb5ae95e464433800025" + } + ] +} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json b/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json new file mode 100644 index 000000000..fb3aede99 --- /dev/null +++ b/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json @@ -0,0 +1,1516 @@ +{ + "schema": "h3-down-measured-case-part-v1", + "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", + "case_indexes": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], + "cases": [ + { + "rows": 1, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.006100177764892578,"max_error_over_tolerance": 0.067171111702919,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0007508695125579834,"max_error_over_tolerance": 0.0071157063357532024,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.05316777154803276,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.7511414587497711, + "samples_ms": [1.3634003698825836, 0.8582398295402527, 0.5834773182868958, 0.7511414587497711, 0.5164369940757751], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2127029001712799, + "samples_ms": [0.2271793782711029, 0.22742152214050293, 0.21224841475486755, 0.21011754870414734, 0.2127029001712799], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.19888952374458313, + "samples_ms": [0.2205371856689453, 0.20452961325645447, 0.19888952374458313, 0.19580498337745667, 0.19642338156700134], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.2423345148563385, + "samples_ms": [4.046555608510971, 4.976723343133926, 4.653695970773697, 3.2874606549739838, 4.2423345148563385], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08771941065788269, + "samples_ms": [0.10984763503074646, 0.09224936366081238, 0.08678808808326721, 0.08628144860267639, 0.08771941065788269], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8339323103427887, + "samples_ms": [0.9573549032211304, 0.6110593676567078, 0.8751153945922852, 0.8339323103427887, 0.4051104187965393], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921","cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48","cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 1,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb", + "dtype": "torch.bfloat16", + "shape": [1, 5376] + }, + "dx": { + "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43", + "dtype": "torch.bfloat16", + "shape": [1, 14336] + }, + "dw": { + "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 31, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00782012939453125,"max_error_over_tolerance": 0.08643997460603714,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009520649909973145,"max_error_over_tolerance": 0.009015342220664024,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078125,"max_error_over_tolerance": 0.055741358548402786,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.49997493624687195, + "samples_ms": [0.5282871425151825, 0.5064867436885834, 0.49997493624687195, 0.49972906708717346, 0.4875659942626953], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.20970404148101807, + "samples_ms": [0.23807212710380554, 0.2144239842891693, 0.20970404148101807, 0.20581483840942383, 0.20891427993774414], + "peak_allocated_bytes": 224250880, + "incremental_peak_bytes": 888832, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.20439177751541138, + "samples_ms": [0.2192482352256775, 0.20439177751541138, 0.21205097436904907, 0.19450858235359192, 0.1872330904006958], + "peak_allocated_bytes": 378764288, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.554342478513718, + "samples_ms": [4.593264311552048, 4.323776811361313, 4.380952566862106, 4.554342478513718, 4.684560000896454], + "peak_allocated_bytes": 537462784, + "incremental_peak_bytes": 314100736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.09334459900856018, + "samples_ms": [0.10614469647407532, 0.096101313829422, 0.09334459900856018, 0.09072944521903992, 0.09304285049438477], + "peak_allocated_bytes": 223695360, + "incremental_peak_bytes": 333312, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.1095404624938965, + "samples_ms": [1.302909106016159, 1.3855025172233582, 1.1095404624938965, 0.4470720887184143, 0.7269084453582764], + "peak_allocated_bytes": 378724864, + "incremental_peak_bytes": 155362816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 31,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6a603f6d21ea02cd5963cf81bb7d1446bd96af21989e0ef649a314e6f195ef24", + "dtype": "torch.bfloat16", + "shape": [31, 5376] + }, + "dx": { + "sha256": "3caf4cfab06db14f91702f774666b715c08cfc28c44b0d1a27ec1fb896b39df2", + "dtype": "torch.bfloat16", + "shape": [31, 14336] + }, + "dw": { + "sha256": "a9e413a88de121e78c0b957b14a4fffbd905590c5646e1905db3ee2e4588f611", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00782012939453125,"max_error_over_tolerance": 0.08643997460603714,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009520649909973145,"max_error_over_tolerance": 0.009015342220664024,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007812023162841797,"max_error_over_tolerance": 0.055688682943582535,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.472191721200943, + "samples_ms": [0.5241893231868744, 0.4849061369895935, 0.472191721200943, 0.4712902009487152, 0.46755746006965637], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19920989871025085, + "samples_ms": [0.2119503915309906, 0.20406395196914673, 0.19920989871025085, 0.1966916024684906, 0.19863247871398926], + "peak_allocated_bytes": 224347648, + "incremental_peak_bytes": 917504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.14410540461540222, + "samples_ms": [0.1502782106399536, 0.14410540461540222, 0.15384703874588013, 0.13763830065727234, 0.13315677642822266], + "peak_allocated_bytes": 377570816, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 2.8609633445739746, + "samples_ms": [2.8609633445739746, 3.24355810880661, 2.8083324432373047, 2.8740428388118744, 2.657875418663025], + "peak_allocated_bytes": 536416256, + "incremental_peak_bytes": 312986112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08560344576835632, + "samples_ms": [0.09668990969657898, 0.08917227387428284, 0.08560344576835632, 0.08519366383552551, 0.08498504757881165], + "peak_allocated_bytes": 223774208, + "incremental_peak_bytes": 344064, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8179731667041779, + "samples_ms": [0.8179731667041779, 0.9407177567481995, 0.8269548416137695, 0.36824867129325867, 0.30737370252609253], + "peak_allocated_bytes": 378832384, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 32,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "c6c699d786504292156fc35e27dcdaab91185830934b117913bfa8e4aff20516", + "dtype": "torch.bfloat16", + "shape": [32, 5376] + }, + "dx": { + "sha256": "56f6ad0df9ec971093c734e6467f691f20a9e46408f53616fb00badb1ae2a60b", + "dtype": "torch.bfloat16", + "shape": [32, 14336] + }, + "dw": { + "sha256": "a980324f5122698c24c2e24cb8cc855c9f698ad7cb657c9e19e06077ab272bf5", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 33, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007819890975952148,"max_error_over_tolerance": 0.08643733710050583,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009521245956420898,"max_error_over_tolerance": 0.009015905670821667,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.014435768127441406,"max_error_over_tolerance": 0.07634487003087997,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5420371890068054, + "samples_ms": [0.508435070514679, 0.6049051880836487, 0.5784481763839722, 0.4877038300037384, 0.5420371890068054], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.1995675265789032, + "samples_ms": [0.20760297775268555, 0.1995675265789032, 0.19385293126106262, 0.2023652195930481, 0.1921132206916809], + "peak_allocated_bytes": 224444928, + "incremental_peak_bytes": 946176, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.19453838467597961, + "samples_ms": [0.21283701062202454, 0.19798427820205688, 0.1898147165775299, 0.19311904907226562, 0.19453838467597961], + "peak_allocated_bytes": 380162560, + "incremental_peak_bytes": 156663808, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.123970866203308, + "samples_ms": [4.327472299337387, 4.013523459434509, 4.435192793607712, 3.1566955149173737, 4.123970866203308], + "peak_allocated_bytes": 539156480, + "incremental_peak_bytes": 315657728, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08546561002731323, + "samples_ms": [0.09231641888618469, 0.08748844265937805, 0.08546561002731323, 0.08390471339225769, 0.08344277739524841], + "peak_allocated_bytes": 223853568, + "incremental_peak_bytes": 354816, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2046098709106445, + "samples_ms": [1.2046098709106445, 1.2146830558776855, 1.2108273804187775, 1.1087283492088318, 0.4986859858036041], + "peak_allocated_bytes": 378940416, + "incremental_peak_bytes": 155441664, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [1,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [1,224]},"canonical_active_rows": 33,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e", + "dtype": "torch.bfloat16", + "shape": [33, 5376] + }, + "dx": { + "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60", + "dtype": "torch.bfloat16", + "shape": [33, 14336] + }, + "dw": { + "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 127, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00780034065246582,"max_error_over_tolerance": 0.08598984032869339,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009499192237854004,"max_error_over_tolerance": 0.009035039693117142,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015492439270019531,"max_error_over_tolerance": 0.08494006842374802,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4916340112686157, + "samples_ms": [0.5290843546390533, 0.5001910030841827, 0.4916340112686157, 0.48222020268440247, 0.47730281949043274], + "peak_allocated_bytes": 231722496, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2099648118019104, + "samples_ms": [0.21921470761299133, 0.2099648118019104, 0.215776264667511, 0.205211341381073, 0.20548328757286072], + "peak_allocated_bytes": 233998336, + "incremental_peak_bytes": 3641344, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.23742765188217163, + "samples_ms": [0.25482848286628723, 0.24598091840744019, 0.23742765188217163, 0.23298710584640503, 0.2289116382598877], + "peak_allocated_bytes": 389959680, + "incremental_peak_bytes": 159602688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.725195467472076, + "samples_ms": [5.029212683439255, 4.744764417409897, 4.252210259437561, 4.3155737221241, 4.725195467472076], + "peak_allocated_bytes": 563719680, + "incremental_peak_bytes": 333362688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.10073184967041016, + "samples_ms": [0.10984763503074646, 0.10212883353233337, 0.10073184967041016, 0.09812414646148682, 0.09807571768760681], + "peak_allocated_bytes": 231722496, + "incremental_peak_bytes": 1365504, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.1009685695171356, + "samples_ms": [1.3191774487495422, 0.9455867111682892, 1.1009685695171356, 0.9224340319633484, 1.1019520461559296], + "peak_allocated_bytes": 389504512, + "incremental_peak_bytes": 159147520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 127,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "69de6ffba99e13cb0f50b6019f6ab4a60222501650d11de89b0c0d0844a0c7ad", + "dtype": "torch.bfloat16", + "shape": [127, 5376] + }, + "dx": { + "sha256": "697a3795ae80f6e5b4b014270998def2b4763acb973ab8134213cb80f19e8fb8", + "dtype": "torch.bfloat16", + "shape": [127, 14336] + }, + "dw": { + "sha256": "f4999860992d23b6ab509cdb088ac0c52df6e23763b7ab78446b19cc94fb2789", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 128, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00780034065246582,"max_error_over_tolerance": 0.08598984032869339,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009499192237854004,"max_error_over_tolerance": 0.009035039693117142,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015538215637207031,"max_error_over_tolerance": 0.0854954943060875,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6464831531047821, + "samples_ms": [0.705275684595108, 0.6767474114894867, 0.6464831531047821, 0.6321556866168976, 0.6436444818973541], + "peak_allocated_bytes": 231804928, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.27082115411758423, + "samples_ms": [0.30130520462989807, 0.2724193036556244, 0.26627257466316223, 0.26065483689308167, 0.27082115411758423], + "peak_allocated_bytes": 234098688, + "incremental_peak_bytes": 3670016, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.24049729108810425, + "samples_ms": [0.2527572214603424, 0.24078786373138428, 0.23163482546806335, 0.23698434233665466, 0.24049729108810425], + "peak_allocated_bytes": 384569344, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.078967660665512, + "samples_ms": [4.93263453245163, 5.035568028688431, 5.260847508907318, 5.078967660665512, 5.102559924125671], + "peak_allocated_bytes": 558902784, + "incremental_peak_bytes": 328474112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11829286813735962, + "samples_ms": [0.13687461614608765, 0.12293830513954163, 0.11829286813735962, 0.11602789163589478, 0.11521577835083008], + "peak_allocated_bytes": 231804928, + "incremental_peak_bytes": 1376256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.49502232670784, + "samples_ms": [1.6067475080490112, 1.5815980732440948, 1.4262758195400238, 1.0990574955940247, 1.49502232670784], + "peak_allocated_bytes": 389615616, + "incremental_peak_bytes": 159186944, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [2,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [2,224]},"canonical_active_rows": 128,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "ba7eaec4960966c7b2a37c6e44c020e67304a0f7877a159fd000c8878500dd90", + "dtype": "torch.bfloat16", + "shape": [128, 5376] + }, + "dx": { + "sha256": "1dc23d75e50aef565ad84ae3dc21d07806176a689e5b906ee775724a71f65a9d", + "dtype": "torch.bfloat16", + "shape": [128, 14336] + }, + "dw": { + "sha256": "f21ca96fd9b83e2a051392a250ce2f39862555f517c3e961445047c5873ee24f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 129, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0078012943267822266,"max_error_over_tolerance": 0.08598456531763077,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009499192237854004,"max_error_over_tolerance": 0.009035039693117142,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015491485595703125,"max_error_over_tolerance": 0.08537446707487106,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.6536431610584259, + "samples_ms": [0.8461922407150269, 0.6669200956821442, 0.6536431610584259, 0.6260201334953308, 0.6349869072437286], + "peak_allocated_bytes": 231888896, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.2928338944911957, + "samples_ms": [0.31632184982299805, 0.2928338944911957, 0.29468536376953125, 0.2854093909263611, 0.2806149423122406], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 3698688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.3973469138145447, + "samples_ms": [0.40783360600471497, 0.38995593786239624, 0.3815554082393646, 0.3973469138145447, 0.48520416021347046], + "peak_allocated_bytes": 391910912, + "incremental_peak_bytes": 161409024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.093999207019806, + "samples_ms": [5.492702126502991, 5.749542266130447, 4.561234265565872, 5.093999207019806, 5.001761019229889], + "peak_allocated_bytes": 565443584, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.15628337860107422, + "samples_ms": [0.18086284399032593, 0.1616552472114563, 0.15628337860107422, 0.1537799835205078, 0.15249475836753845], + "peak_allocated_bytes": 231888896, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.5014559030532837, + "samples_ms": [1.3852529227733612, 1.226574182510376, 1.5177838504314423, 1.5014559030532837, 1.6144178807735443], + "peak_allocated_bytes": 389728256, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [3,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [3,224]},"canonical_active_rows": 129,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236", + "dtype": "torch.bfloat16", + "shape": [129, 5376] + }, + "dx": { + "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8", + "dtype": "torch.bfloat16", + "shape": [129, 14336] + }, + "dw": { + "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 256, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007821321487426758,"max_error_over_tolerance": 0.08593791723251343,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009765326976776123,"max_error_over_tolerance": 0.009236739948391914,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.015624523162841797,"max_error_over_tolerance": 0.08663392066955566,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5091354250907898, + "samples_ms": [0.594262033700943, 0.5234144628047943, 0.5091354250907898, 0.5043074488639832, 0.5023069679737091], + "peak_allocated_bytes": 242360320, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.23020803928375244, + "samples_ms": [0.24154409766197205, 0.24346262216567993, 0.23020803928375244, 0.22159144282341003, 0.22471323609352112], + "peak_allocated_bytes": 246947840, + "incremental_peak_bytes": 7340032, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.25147944688796997, + "samples_ms": [0.2661608159542084, 0.25453418493270874, 0.2458915114402771, 0.25030598044395447, 0.25147944688796997], + "peak_allocated_bytes": 393748480, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 3.2671354711055756, + "samples_ms": [2.359338104724884, 2.3001059889793396, 3.2671354711055756, 3.9541684091091156, 3.9238929748535156], + "peak_allocated_bytes": 586439168, + "incremental_peak_bytes": 346831360, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1126602292060852, + "samples_ms": [0.12792646884918213, 0.11543557047843933, 0.1126602292060852, 0.11115893721580505, 0.11189281940460205], + "peak_allocated_bytes": 242360320, + "incremental_peak_bytes": 2752512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.8542537689208984, + "samples_ms": [0.9875744581222534, 0.8467361330986023, 0.9040683507919312, 0.8542537689208984, 0.4503205418586731], + "peak_allocated_bytes": 403841024, + "incremental_peak_bytes": 164233216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [4,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [4,224]},"canonical_active_rows": 256,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "948b289d0ffaf1e67d6aad8e9ae565c21a57ed5809a56531cd802a673a7a1186", + "dtype": "torch.bfloat16", + "shape": [256, 5376] + }, + "dx": { + "sha256": "d5f3c01d93a8a90120e0c09a1f7706435e4a615acc1dff5b5e32bd14bdc6d790", + "dtype": "torch.bfloat16", + "shape": [256, 14336] + }, + "dw": { + "sha256": "71cd2c2acd3131879911a8cd2c79299f560f0a160c28e2d79a0a67272b8fd152", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 513, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00783395767211914,"max_error_over_tolerance": 0.08669471740722656,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009765625,"max_error_over_tolerance": 0.009295407682657242,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.03013134002685547,"max_error_over_tolerance": 0.11083704233169556,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5357936024665833, + "samples_ms": [0.5600899457931519, 0.52647665143013, 0.5357936024665833, 0.5212984979152679, 0.7661432027816772], + "peak_allocated_bytes": 262334464, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.5339905619621277, + "samples_ms": [0.6604790687561035, 0.5883835256099701, 0.5339905619621277, 0.375404953956604, 0.3919526934623718], + "peak_allocated_bytes": 271527424, + "incremental_peak_bytes": 14708736, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.4009269177913666, + "samples_ms": [0.6171725690364838, 0.4009269177913666, 0.39183348417282104, 0.39962679147720337, 0.4404895007610321], + "peak_allocated_bytes": 432406016, + "incremental_peak_bytes": 175587328, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 2.8889253735542297, + "samples_ms": [2.820633351802826, 2.8889253735542297, 2.8440728783607483, 2.9198452830314636, 3.1279921531677246], + "peak_allocated_bytes": 661960704, + "incremental_peak_bytes": 405142016, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.1828409731388092, + "samples_ms": [0.19228458404541016, 0.1829490065574646, 0.18090009689331055, 0.17902255058288574, 0.1828409731388092], + "peak_allocated_bytes": 262334464, + "incremental_peak_bytes": 5515776, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 0.7188506424427032, + "samples_ms": [0.7435493171215057, 0.7229670882225037, 0.7097125053405762, 0.7188506424427032, 0.6750933825969696], + "peak_allocated_bytes": 431183872, + "incremental_peak_bytes": 174365184, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [9,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [9,224]},"canonical_active_rows": 513,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "305525b6b8360ad1215647dc5b4412b9cf085864533f03ffc40ecb9cb77555ea", + "dtype": "torch.bfloat16", + "shape": [513, 5376] + }, + "dx": { + "sha256": "8bb06fd1bbe46d5a333c3e50e1906131da3f6e9a8049479f7595a20ccb79a1cf", + "dtype": "torch.bfloat16", + "shape": [513, 14336] + }, + "dw": { + "sha256": "54314948f05a018ef73b260df92cfb432d1fca3506a865f9ff33f9243878b401", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 1024, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007846832275390625,"max_error_over_tolerance": 0.08643604815006256,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009739995002746582,"max_error_over_tolerance": 0.009267088025808334,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.031244277954101562,"max_error_over_tolerance": 0.11963395029306412,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.003596931695938, + "samples_ms": [1.0749511420726776, 1.0326988995075226, 0.9927935898303986, 0.9920410811901093, 1.003596931695938], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.6674230098724365, + "samples_ms": [0.6864443421363831, 0.6674230098724365, 0.6535425782203674, 0.6415434181690216, 0.6711967289447784], + "peak_allocated_bytes": 320372736, + "incremental_peak_bytes": 29360128, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.5908161401748657, + "samples_ms": [0.6029978394508362, 0.5908161401748657, 0.593278557062149, 0.5755200982093811, 0.5740784108638763], + "peak_allocated_bytes": 445153280, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.697073251008987, + "samples_ms": [4.6738311648368835, 4.697073251008987, 4.740521311759949, 4.760138690471649, 4.647858440876007], + "peak_allocated_bytes": 749822976, + "incremental_peak_bytes": 458810368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.27285143733024597, + "samples_ms": [0.2916380763053894, 0.2776235342025757, 0.27285143733024597, 0.26857107877731323, 0.2683289349079132], + "peak_allocated_bytes": 302022656, + "incremental_peak_bytes": 11010048, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.4369338750839233, + "samples_ms": [1.4369338750839233, 1.3857372105121613, 1.5310943126678467, 1.4232508838176727, 1.5554651618003845], + "peak_allocated_bytes": 485523456, + "incremental_peak_bytes": 194510848, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [16,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [16,224]},"canonical_active_rows": 1024,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "5a8c854fa60235b480d333b01b7017b1319c0daf54dff2338dc625018fecaf70", + "dtype": "torch.bfloat16", + "shape": [1024, 5376] + }, + "dx": { + "sha256": "208e5216e4a16a5accf3a50ad3ef4fba4ff062322f0fb8dd1b4b364389095238", + "dtype": "torch.bfloat16", + "shape": [1024, 14336] + }, + "dw": { + "sha256": "970175a5ac5a8e206233200b0865d3df694adf5888d5b7b441ce14b0859f270b", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 2048, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007836103439331055,"max_error_over_tolerance": 0.08683990687131882,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009764134883880615,"max_error_over_tolerance": 0.00927541870623827,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.031280517578125,"max_error_over_tolerance": 0.11995884031057358,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 1.4924593269824982, + "samples_ms": [1.575138419866562, 1.537363976240158, 1.4924593269824982, 1.4637857675552368, 1.4722123742103577], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 1.1428110301494598, + "samples_ms": [1.1428110301494598, 1.1433251202106476, 1.1936947703361511, 1.1115595698356628, 1.0797232389450073], + "peak_allocated_bytes": 419495936, + "incremental_peak_bytes": 58720256, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 1.0681450366973877, + "samples_ms": [1.0858029127120972, 1.0655000805854797, 1.0727979242801666, 1.0573789477348328, 1.0681450366973877], + "peak_allocated_bytes": 514916352, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 6.326321512460709, + "samples_ms": [5.975883454084396, 6.353158503770828, 6.0294009745121, 6.326321512460709, 13.371232897043228], + "peak_allocated_bytes": 970115072, + "incremental_peak_bytes": 609339392, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.5012601613998413, + "samples_ms": [0.49800798296928406, 0.5012601613998413, 0.5335211753845215, 0.5161166191101074, 0.49573928117752075], + "peak_allocated_bytes": 382795776, + "incremental_peak_bytes": 22020096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.7548240721225739, + "samples_ms": [1.710943877696991, 1.753535121679306, 1.828797161579132, 1.8369108438491821, 1.7548240721225739], + "peak_allocated_bytes": 595656704, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [32,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [32,224]},"canonical_active_rows": 2048,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "4fc22b3ef0763c90709fdcf83d304de53076d9e5010a263e1b34d9d6cfe59be4", + "dtype": "torch.bfloat16", + "shape": [2048, 5376] + }, + "dx": { + "sha256": "a0ef242ffebeb3b2b716559c6c25462059938b7b56e791c2d21a80be9e4f9005", + "dtype": "torch.bfloat16", + "shape": [2048, 14336] + }, + "dw": { + "sha256": "da2b0000f22f1145ff004e49eeab6c826babe129a04f3890576321d747b58075", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 4096, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007841348648071289,"max_error_over_tolerance": 0.08670263737440109,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009782612323760986,"max_error_over_tolerance": 0.009300120174884796,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06252670288085938,"max_error_over_tolerance": 0.14839504659175873,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 2.8487443923950195, + "samples_ms": [2.87516787648201, 2.8021037578582764, 2.834375947713852, 2.8487443923950195, 2.903975546360016], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 2.128344029188156, + "samples_ms": [2.1580159664154053, 2.1231621503829956, 2.089381217956543, 2.128344029188156, 2.1305643022060394], + "peak_allocated_bytes": 617742336, + "incremental_peak_bytes": 117440512, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 2.0004957914352417, + "samples_ms": [2.03578919172287, 2.0004957914352417, 2.0807236433029175, 1.9929297268390656, 1.990877091884613], + "peak_allocated_bytes": 654442496, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 9.617339819669724, + "samples_ms": [9.428001940250397, 9.563304483890533, 9.74905863404274, 10.471254587173462, 9.617339819669724], + "peak_allocated_bytes": 1410699264, + "incremental_peak_bytes": 910397440, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.9003691375255585, + "samples_ms": [1.0827071964740753, 1.191694289445877, 0.9003691375255585, 0.8887462317943573, 0.8913837373256683], + "peak_allocated_bytes": 544342016, + "incremental_peak_bytes": 44040192, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 2.7191415429115295, + "samples_ms": [2.706039696931839, 2.7057379484176636, 2.7206838130950928, 2.719506621360779, 2.7191415429115295], + "peak_allocated_bytes": 815923200, + "incremental_peak_bytes": 315621376, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [64,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [64,224]},"canonical_active_rows": 4096,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "a60e34372727dd9149217014270081afc07bd4915d95e4a7a85a237c08a4b178", + "dtype": "torch.bfloat16", + "shape": [4096, 5376] + }, + "dx": { + "sha256": "a74718d8c858efb3bac5fc50ba60010b8882e12a46510a93dc2ede260e01296c", + "dtype": "torch.bfloat16", + "shape": [4096, 14336] + }, + "dw": { + "sha256": "a96f78202954c57a84b1acc245f5d8131a12a9f7d9ffef98bc41847715d4167c", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 8192, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007861852645874023,"max_error_over_tolerance": 0.08676071465015411,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009774267673492432,"max_error_over_tolerance": 0.0092932665720582,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.06272697448730469,"max_error_over_tolerance": 0.14868396520614624,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 6.731804460287094, + "samples_ms": [6.731804460287094, 6.341461092233658, 6.911188364028931, 6.427716463804245, 6.800435483455658], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 4.341848194599152, + "samples_ms": [4.340160638093948, 4.464488476514816, 4.254065454006195, 4.358276724815369, 4.341848194599152], + "peak_allocated_bytes": 1014235136, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 4.210811108350754, + "samples_ms": [4.122566431760788, 4.205178469419479, 4.225626587867737, 4.210811108350754, 4.668742418289185], + "peak_allocated_bytes": 933494784, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 19.256968051195145, + "samples_ms": [19.256968051195145, 18.107809126377106, 19.30183172225952, 19.30096000432968, 19.102156162261963], + "peak_allocated_bytes": 2291867648, + "incremental_peak_bytes": 1512513536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 1.716524362564087, + "samples_ms": [1.8020570278167725, 1.7209723591804504, 1.716524362564087, 1.7112195491790771, 1.7103217542171478], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 5.995567888021469, + "samples_ms": [5.205079913139343, 5.2115097641944885, 5.995567888021469, 6.085146218538284, 8.324682712554932], + "peak_allocated_bytes": 1256456192, + "incremental_peak_bytes": 477102080, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [128,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [128,224]},"canonical_active_rows": 8192,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "65c1cf45a27c17a87b82850851f40e8e8e91d13925ee21882e513b8dcb1d4d0c", + "dtype": "torch.bfloat16", + "shape": [8192, 5376] + }, + "dx": { + "sha256": "5c42594fdd1a798ef93cd0bfce380ed91ed081746c84b269bff4008eb26b580e", + "dtype": "torch.bfloat16", + "shape": [8192, 14336] + }, + "dw": { + "sha256": "073de23aa27b3c4b2b8f3a33d7b249d2ad529fe1883ac3676370a88ac7601362", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 16385, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007851600646972656,"max_error_over_tolerance": 0.08705637603998184,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009774565696716309,"max_error_over_tolerance": 0.009300513193011284,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12546157836914062,"max_error_over_tolerance": 0.16740718483924866,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 20.043708384037018, + "samples_ms": [20.18143981695175, 20.07604017853737, 19.939277321100235, 19.863557070493698, 20.043708384037018], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 8.881796151399612, + "samples_ms": [9.00673121213913, 8.875839412212372, 8.893750607967377, 8.881796151399612, 8.768562227487564], + "peak_allocated_bytes": 1807319040, + "incremental_peak_bytes": 469790720, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 9.493160992860794, + "samples_ms": [9.27766039967537, 9.493160992860794, 9.644988924264908, 9.2148557305336, 9.505551308393478], + "peak_allocated_bytes": 2138853376, + "incremental_peak_bytes": 801325056, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 43.05011406540871, + "samples_ms": [43.04959625005722, 43.427929282188416, 43.05969923734665, 42.90411248803139, 43.05011406540871], + "peak_allocated_bytes": 4701607936, + "incremental_peak_bytes": 3364079616, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 3.1884387135505676, + "samples_ms": [3.1987205147743225, 3.1884387135505676, 3.173545002937317, 3.177415579557419, 3.2145678997039795], + "peak_allocated_bytes": 1513699840, + "incremental_peak_bytes": 176171520, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 11.09558716416359, + "samples_ms": [10.03454253077507, 10.854113847017288, 11.104598641395569, 11.09558716416359, 11.109769344329834], + "peak_allocated_bytes": 2137631232, + "incremental_peak_bytes": 800102912, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [257,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [257,224]},"canonical_active_rows": 16385,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "801986e1776078be2cbc69588869fc45075cd3c8a1110e8ef659d6af8c6a6e66", + "dtype": "torch.bfloat16", + "shape": [16385, 5376] + }, + "dx": { + "sha256": "aa68ce734273a068db9d32d95009990e8845a2287544e6bb806309bed9c68f5b", + "dtype": "torch.bfloat16", + "shape": [16385, 14336] + }, + "dw": { + "sha256": "b1f7ef34a54388d33cba9e61ece11f1e395f32394226c44ea8eae2c8121554ff", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 24577, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.00786590576171875,"max_error_over_tolerance": 0.08697982132434845,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.000977933406829834,"max_error_over_tolerance": 0.009302553720772266,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12613296508789062,"max_error_over_tolerance": 0.16935373842716217,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "inactive_padding"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "inactive_padding"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 31.051654368638992, + "samples_ms": [30.949130654335022, 31.17082267999649, 31.01911023259163, 31.051654368638992, 32.34020248055458], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 18.86589080095291, + "samples_ms": [19.044052809476852, 18.86589080095291, 17.993338406085968, 18.721047788858414, 21.820947527885437], + "peak_allocated_bytes": 2600304640, + "incremental_peak_bytes": 704671744, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 14.459915459156036, + "samples_ms": [14.484800398349762, 14.459915459156036, 14.379594475030899, 14.104023575782776, 14.869511127471924], + "peak_allocated_bytes": 3019919360, + "incremental_peak_bytes": 1124286464, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 70.6416592001915, + "samples_ms": [70.6416592001915, 71.31434231996536, 71.67328521609306, 69.40366700291634, 70.14808431267738], + "peak_allocated_bytes": 6786906112, + "incremental_peak_bytes": 4891273216, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 5.352184176445007, + "samples_ms": [5.024075508117676, 5.151830613613129, 5.389474332332611, 5.390428006649017, 5.352184176445007], + "peak_allocated_bytes": 2159884800, + "incremental_peak_bytes": 264251904, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 18.56180652976036, + "samples_ms": [17.890069633722305, 18.56180652976036, 19.248690456151962, 18.597692251205444, 18.478691577911377], + "peak_allocated_bytes": 3018697216, + "incremental_peak_bytes": 1123064320, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [385,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [385,224]},"canonical_active_rows": 24577,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "9095fd83587eea976861650c3dc4edabd12321eb89a8f8dea2f530b499f1b058", + "dtype": "torch.bfloat16", + "shape": [24577, 5376] + }, + "dx": { + "sha256": "97f01ad55ca2b6158cde2dc3e8756a7d46bd97ea16b0611bd97b7e22355ffcba", + "dtype": "torch.bfloat16", + "shape": [24577, 14336] + }, + "dw": { + "sha256": "3cee55c58793b1a9920e6bbdd266cfe9d8ab6ae1cd66f439129f5b214309302f", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + }, + { + "rows": 32768, + "input_family": "swiglu_like", + "checks": [ + {"judgment": "forward_accuracy","tolerance": {"judgment": "forward_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.05,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.007860183715820312,"max_error_over_tolerance": 0.08708541095256805,"failing_elements": 0,"tensor": "y","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.0009776651859283447,"max_error_over_tolerance": 0.009305901825428009,"failing_elements": 0,"tensor": "dx","transform": "fp32_reference"}, + {"judgment": "gradient_accuracy","tolerance": {"judgment": "gradient_accuracy","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "tolerance","atol": 0.1,"rtol": 0.02,"comparison_lhs_role": "bf16_candidate","comparison_rhs_role": "fp32_reference","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"max_abs_error": 0.12707138061523438,"max_error_over_tolerance": 0.1699322909116745,"failing_elements": 0,"tensor": "dw","transform": "fp32_reference"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "repeat"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "repeat"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "permuted_logical_rows"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "permuted_logical_rows"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "chunk31_single_backward"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "chunk31_single_backward"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "strided_input_and_weight"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "strided_input_and_weight"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "batch2_reshape"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dw","transform": "batch2_reshape"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "singleton_vs_batch"}, + {"judgment": "forward_invariance","tolerance": {"judgment": "forward_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "y","transform": "other_rows_mutated"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "singleton_vs_batch"}, + {"judgment": "gradient_invariance","tolerance": {"judgment": "gradient_invariance","op_class": "reduction","dtype_name": "bfloat16","status": "applicable","mode": "bitwise","atol": 0.0,"rtol": 0.0,"comparison_lhs_role": "transformed_config","comparison_rhs_role": "canonical_config","backend_profile": "cuda_bf16","arch_key": "sm90"},"passed": true,"mismatched_bytes": 0,"tensor": "dx","transform": "other_rows_mutated"} + ], + "benchmarks": { + "candidate_forward": { + "median_ms": 41.59888997673988, + "samples_ms": [41.66906327009201, 41.59888997673988, 41.56194254755974, 41.601087898015976, 41.56932234764099], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 37.15309500694275, + "samples_ms": [36.36578842997551, 37.498798221349716, 37.37569972872734, 37.15309500694275, 36.31233796477318], + "peak_allocated_bytes": 3393191936, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 18.57171207666397, + "samples_ms": [20.10498195886612, 19.960522651672363, 18.57171207666397, 18.499113619327545, 18.54676753282547], + "peak_allocated_bytes": 2607808512, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 108.27267915010452, + "samples_ms": [108.27267915010452, 109.35502871870995, 108.98803547024727, 105.77517747879028, 106.29720613360405], + "peak_allocated_bytes": 7578877952, + "incremental_peak_bytes": 5125210112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 7.173832505941391, + "samples_ms": [6.3136667013168335, 6.678733974695206, 7.226929068565369, 7.173832505941391, 7.515721023082733], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 21.348431706428528, + "samples_ms": [22.704727947711945, 21.976932883262634, 20.644009113311768, 21.064143627882004, 21.348431706428528], + "peak_allocated_bytes": 3899654144, + "incremental_peak_bytes": 1445986304, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + }, + "execution_traces": [ + {"requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1","kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","fallback": false,"arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast","tile": [64,64,32],"num_warps": 4,"num_stages": 2,"fp_fusion": false,"split_k": false,"split_policy": "none","accumulator_dtype": "float32","schedule_id": "fixed-64x64x32-w4-s2-fusion0","topology": {"world_size": 1,"tensor_parallel_size": 1},"checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe","weight_layout": "[5376,14336]","dw_order": "lexicographic active logical keys; trailing zero pad to 32","forward_executed": true,"input_gradient_executed": true,"weight_gradient_executed": true,"qualification": "pending_h3_gpu_evidence","forward_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0","cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb","compiled_num_warps": 4,"grid": [512,84]},"input_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455","cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd","compiled_num_warps": 4,"grid": [512,224]},"canonical_active_rows": 32768,"weight_gradient_launch": {"kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel","triton_version": "3.2.0","compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954","cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a","compiled_num_warps": 4,"grid": [84,224]}} + ], + "output_digests": { + "y": { + "sha256": "46b761fcf0a9ab3186d792ed8a5fe420a22a76fd2fe30709fb5a446cad834ea8", + "dtype": "torch.bfloat16", + "shape": [32768, 5376] + }, + "dx": { + "sha256": "8b9866742cfe3ed59b577b6a0f27628052499205e7322bb48a66c70afcafae8d", + "dtype": "torch.bfloat16", + "shape": [32768, 14336] + }, + "dw": { + "sha256": "5ff54c5913b0c7e414a95f29aefcba834e10ed35d837aa01c9b659ec5a4c745d", + "dtype": "torch.bfloat16", + "shape": [5376, 14336] + } + }, + "passed": true + } + ] +} diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md new file mode 100644 index 000000000..bc2343b83 --- /dev/null +++ b/docs/operators/h3-ffn-down.md @@ -0,0 +1,163 @@ +# H3 deterministic FFN down projection + +CUDA WS1 candidate for `zzf2024`'s `h3_ffn_down_gemm` work item in +[issue #420](https://github.com/RL-Align/RL-Kernel/issues/420). +Base: `test-h3@32b765ec992cd1206517104ec66506881203c91c`. +Checkpoint: `MiniMaxAI/MiniMax-H3@42ed227ee7df40d41602854ae760620d6eb651fe`. + +## Contract and scope + +Backend `rlkernel.h3_ffn_down.triton.fp32_tiles.v1` reuses the existing +`triton.matmul.det_gemm._det_gemm_fp32_kernel`. BF16 operands feed FP32 +32-wide dot products and ascending FP32 tile additions, followed by one BF16 +cast. The schedule is fixed at 64x64x32, four warps and two stages, with +`enable_fp_fusion=False`, TF32 disabled, no tuning, no split-K and no fallback. +No new CUDA or Triton device kernel is added. + +The eager operator accepts BF16 `[M,14336]` or `[B,S,14336]`, flattened row count +1..32768, native `[5376,14336]` weights and no bias on one SM90 CUDA device. +It computes `Y=X@W.T`, `dX=dY@W`, and canonical `dW=dY.T@X`. Strided tensors +are normalized by copies. Weight-gradient rows are sorted by logical identity +and padded to a fixed 32-row extent before a single reduction. + +The operator rejects invalid shapes, types, devices, TF32, duplicate active +keys, inconsistent weight identity and nonzero padding gradients. Unsupported +hardware or backend fails closed. Graph compilation, double backward and +retained-graph replay are unsupported. The registry descriptor remains +**test-only**; strict production resolution requires further qualification. +TP, whole-model integration, ROCm and cross-platform equality are outside this PR. + +Inference requires no backward session: + +```python +import torch +from rl_engine.kernels.ops.h3_ffn_down import H3FFNDownGemmOp + +torch.backends.cuda.matmul.allow_tf32 = False +op = H3FFNDownGemmOp() +with torch.no_grad(): + y = op(swiglu_output, down_weight) +print(op.last_execution) +``` + +Training requires one session spanning **all forward uses and one backward call**: + +```python +from rl_engine.kernels.ops.canonical_backward import canonical_backward_session + +with canonical_backward_session() as session: + outputs = [op(x, down_weight, logical_keys=keys, + parameter_id="transformer.blocks.0.ffn.down.weight") + for x, keys in zip(parts, key_parts, strict=True)] + torch.cat(outputs).backward(upstream_gradient) + session.validate_complete() +``` + +Keys are same-device int64 `[rows,2]` or `[rows,3]`, such as +`[sample_id,token_id]`, with stable unique identities across packing, permutation +and microbatching. A negative first component marks inactive padding. The same +weight Tensor must retain one parameter ID. Do not sum independently rounded +microbatch weight gradients; call `validate_complete()` to detect missing branches. +Execution traces record requested/actual backend, kernel, arithmetic, schedule, +topology, checkpoint, executed gradients and actual JIT/cubin hashes. + +## H100 results + +Measured on 2026-10-10: dedicated non-MIG H100 80GB HBM3, capability 9.0, +81,559 MiB, driver 570.211.01, Ubuntu 22.04, Python 3.10.12, torch 2.6.0+cu124, +CUDA 12.4 and primary Triton 3.2.0. + +- **64 GPU/profile tests passed** before PR formatting. +- **49 cases / 1,141 comparisons passed**. Random, synthetic SwiGLU-like and + cancellation inputs cover M=1,31,32,33,127,128,129,256,513,1024,2048,4096, + 8192,16385,24577,32768; a real-weight fixture adds M=129. +- Forward/dX/dW match an independent FP32 PyTorch autograd reference with the + shared `ws1-c1-v2` tolerances. Maximum error/tolerance is **0.18233**, below 1. + No tolerance was weakened. +- Raw-byte checks cover repeats, logical-row permutation, 31-row chunking with + one backward, padding, non-contiguous layouts, batch reshaping, singleton rows + and mutation of unrelated rows. NaN/Inf and signed-zero differences fail. +- A separate process and fresh Triton 3.2.0 cache passed 10 cases with **30 matching + y/dX/dW hashes**. A separately pinned Triton 3.4.0 cache passed five selected + cases with **15 matching hashes**. Both profiles used the same GPU and PyTorch. + +The checkpoint fixture uses `transformer_blocks.0.ff.net.2.weight`, BF16 +`[5376,14336]`, fetched through validated HTTP ranges without downloading the +whole model. Weight SHA256 is +`e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f`. +Activations and upstream gradients are **synthetic**, not captured model data. + +The explicit legacy backend `rlkernel.h3_ffn_down.sm90.midtree.v1` failed real-weight +forward accuracy at M=129: 23,353 failing elements, maximum error/tolerance +5.00441. Its failure is preserved; the BF16 midpoint tree is never selected +as an automatic fallback. + +Five-sample synchronized wall medians for primary-profile random inputs: + +| M | Candidate forward ms | torch BF16 forward ms | Candidate forward+backward ms | torch BF16 forward+backward ms | +|---:|---:|---:|---:|---:| +| 1 | 0.4641 | 0.0822 | 5.8811 | 1.2091 | +| 129 | 0.5118 | 0.1100 | 4.3671 | 1.2572 | +| 8192 | 7.1128 | 1.7194 | 18.4472 | 5.2220 | +| 32768 | 41.6259 | 7.1671 | 107.5935 | 21.5373 | + +These include wrapper checks, copies, allocations and canonical preparation, +not isolated kernel latency. This candidate is slower than ordinary torch BF16. +The comparator has not been qualified for byte invariance. + +Vendored vLLM commit `7797b6022c129b862e45ae6aed08822e65d1bccb` requires the separately +pinned Triton 3.4.0 profile because 3.2.0 rejects `tl.range(..., flatten=True)`. +Its partial probe passed y/dX/dW accuracy and y/dX chunk/singleton invariance. +Within 3.4.0 at random M=32768, vLLM forward took 9.6604 ms versus this adapter's +41.5963 ms; its dW primitive took 40.0210 ms versus 20.9800 ms for fixed FP32. +The vLLM primitive excludes this adapter's checks/copies and has no qualified +canonical autograd session or full dW invariance matrix. It remains prior art. + +## Core JSON evidence + +Only the following measured JSON files accompany this document: + +- [Summary, fixture, tests and rejected native candidate](evidence/h3-down-h100-20261010/h100-summary.json). +- Full matrix: [random](evidence/h3-down-h100-20261010/h100-random.json), + [SwiGLU-like](evidence/h3-down-h100-20261010/h100-swiglu-like.json), and + [cancellation](evidence/h3-down-h100-20261010/h100-cancellation.json). +- [Cold-cache/compiler replay and vLLM comparison](evidence/h3-down-h100-20261010/h100-replays.json). + +The split matrix is lossless: combine the three `cases` arrays with the summary's +`fixture_cases`, sort by their `case_indexes` / `fixture_case_indexes`, and attach +`full_report_header`. That reproduces the original measured report exactly. +Every file is below the repository's 500 KiB pre-commit limit. JSON retains all +checks, tolerances, timing samples, peak memory, output hashes, compiled-kernel +identities and source fingerprints. Original logs, PNG/CSV, source snapshots, +archives and rental details remain local and are excluded from the PR. + +GPU reports identify the measured source. Subsequent Black/isort formatting +is recorded separately with original/current hashes, unchanged non-import AST +and unchanged import bindings; no GPU rerun after formatting is claimed. + +## Reproduction and remaining qualification + +Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: + +```bash +python3.10 -m venv .venv +source .venv/bin/activate +python -m pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 +python -m pip install -r scripts/requirements-h3-down.txt +python scripts/prepare_h3_down_fixture.py --rows 129 --output /workspace/h3-down-fixture.pt +H3_FIXTURE=/workspace/h3-down-fixture.pt bash scripts/run_h3_down_h100.sh smoke +python scripts/validate_h3_ffn_down.py --seed 420 --repeats 5 \ + --rows 1,31,32,33,127,128,129,256,513,1024,2048,4096,8192,16385,24577,32768 \ + --families random,swiglu_like,cancellation \ + --fixture /workspace/h3-down-fixture.pt --output /tmp/h3-down-full.json +``` + +The default needs Triton, not the native extension. `H3_BUILD_NATIVE=1` optionally +builds the legacy comparison with fast math disabled; that build requires matching +Python development headers and nvcc 12.4. `--compare-vllm` is an explicit partial +probe; use a separate Triton 3.4.0 environment without replacing the primary profile. + +This change is ready for **operator review**, with `acceptance_complete=false`. +Captured H3 activations, review of arithmetic/covered layouts, formal runtime +integration and a performance decision remain before production promotion. +It does not close the full H3 roadmap or assert portable equality across devices. diff --git a/rl_engine/kernels/ops/h3_down_fp32_backend.py b/rl_engine/kernels/ops/h3_down_fp32_backend.py new file mode 100644 index 000000000..fac6d94dd --- /dev/null +++ b/rl_engine/kernels/ops/h3_down_fp32_backend.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""H3 adapter for the existing fixed-order Triton FP32 tile GEMM kernel.""" + +import hashlib + +import torch + +_BINARY_HASHES = {} + + +class FixedFP32Backend: + """BF16 operands; FP32 dot tiles and ordered FP32 additions; one final cast. + + Reuses `_det_gemm_fp32_kernel`, bypassing its FP32-input/output wrapper. + Fixed 64x64x32 tiles, four warps, two stages, no tuning or split-K. + """ + + h3_kernel_id = "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel" + + def __init__(self): + from rl_engine.kernels.ops.triton.matmul import det_gemm + + if not det_gemm._TRITON_AVAILABLE: + raise RuntimeError("Triton is required for H3 FP32 tiles; fallback is forbidden") + self.kernel = det_gemm._det_gemm_fp32_kernel + self.triton = det_gemm.triton + + def gemm(self, a, b): + m, k = a.shape + n = b.shape[1] + output = torch.empty((m, n), dtype=torch.bfloat16, device=a.device) + compiled = self.kernel[(self.triton.cdiv(m, 64), self.triton.cdiv(n, 64))]( + a, + b, + output, + m, + n, + k, + *a.stride(), + *b.stride(), + *output.stride(), + BLOCK_M=64, + BLOCK_N=64, + BLOCK_K=32, + PROMOTE_INPUTS=False, + num_warps=4, + num_stages=2, + enable_fp_fusion=False, + ) + if compiled.hash not in _BINARY_HASHES: + _BINARY_HASHES[compiled.hash] = hashlib.sha256(compiled.asm["cubin"]).hexdigest() + self.h3_last_launch = { + "kernel_id": self.h3_kernel_id, + "triton_version": self.triton.__version__, + "compiled_kernel_hash": compiled.hash, + "cubin_sha256": _BINARY_HASHES[compiled.hash], + "compiled_num_warps": compiled.metadata.num_warps, + "grid": [self.triton.cdiv(m, 64), self.triton.cdiv(n, 64)], + } + return output + + def det_gemm_fwd_rhs_transposed(self, x, weight): + return self.gemm(x, weight.t()) + + def det_gemm_fwd(self, gradient, weight): + return self.gemm(gradient, weight) + + def det_gemm_db_transposed(self, x, gradient): + return self.gemm(gradient.t(), x) diff --git a/rl_engine/kernels/ops/h3_down_validation.py b/rl_engine/kernels/ops/h3_down_validation.py new file mode 100644 index 000000000..6f73a1f8d --- /dev/null +++ b/rl_engine/kernels/ops/h3_down_validation.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Independent references and four-judgment evidence for H3 down projection.""" + +from __future__ import annotations + +from contextlib import contextmanager + +import torch + +from rl_engine.kernels.gtest.tolerance import load_contract, resolve_tolerance + + +@contextmanager +def fp32_math(): + old_tf32 = torch.backends.cuda.matmul.allow_tf32 + old_precision = torch.get_float32_matmul_precision() + try: + torch.set_float32_matmul_precision("highest") + torch.backends.cuda.matmul.allow_tf32 = False + with torch.autocast("cuda", enabled=False), torch.autocast("cpu", enabled=False): + yield + finally: + torch.set_float32_matmul_precision(old_precision) + torch.backends.cuda.matmul.allow_tf32 = old_tf32 + + +def reference_projection(x, weight, grad_output): + """Autograd FP32 oracle from the SAME quantized inputs, retaining FP32 results.""" + with fp32_math(), torch.enable_grad(): + ref_x = x.detach().float().requires_grad_(True) + ref_weight = weight.detach().float().requires_grad_(True) + output = torch.nn.functional.linear(ref_x, ref_weight) + dx, dw = torch.autograd.grad(output, (ref_x, ref_weight), grad_output.detach().float()) + return tuple(value.detach() for value in (output, dx, dw)) + + +def raw_equal(lhs, rhs): + if lhs.shape != rhs.shape or lhs.dtype != rhs.dtype: + return False + return torch.equal( + lhs.detach().contiguous().reshape(-1).view(torch.uint8), + rhs.detach().contiguous().reshape(-1).view(torch.uint8), + ) + + +def comparison(lhs, rhs, judgment, *, contract=None): + contract = load_contract() if contract is None else contract + spec = resolve_tolerance( + contract, + judgment=judgment, + op_class="reduction", + dtype="bfloat16", + arch_key="sm90", + backend_profile="cuda_bf16", + ) + result = {"judgment": judgment, "tolerance": spec.to_dict(), "passed": False} + if lhs.shape != rhs.shape: + result["error"] = "shape mismatch" + return result + # NaN/Inf hard fail; signed-zero differences count as bitwise mismatches. + if not torch.isfinite(lhs).all().item() or not torch.isfinite(rhs).all().item(): + result["error"] = "nonfinite tensor" + return result + if spec.mode == "bitwise": + result["passed"] = raw_equal(lhs, rhs) + if lhs.dtype == rhs.dtype: + bits_l = lhs.contiguous().reshape(-1).view(torch.uint8) + bits_r = rhs.contiguous().reshape(-1).view(torch.uint8) + result["mismatched_bytes"] = int((bits_l != bits_r).sum().item()) + else: + result["error"] = "dtype mismatch" + return result + # Chunk comparisons to avoid several full-size FP32 dW temporaries. + left, right = lhs.reshape(-1), rhs.reshape(-1) + max_abs = max_ratio = 0.0 + failed = 0 + for lo in range(0, left.numel(), 1 << 20): + l32 = left[lo : lo + (1 << 20)].float() + r32 = right[lo : lo + (1 << 20)].float() + error = (l32 - r32).abs() + limit = spec.atol + spec.rtol * r32.abs() + max_abs = max(max_abs, error.max().item()) + max_ratio = max(max_ratio, (error / limit).max().item()) + failed += int((error > limit).sum().item()) + result.update( + max_abs_error=max_abs, + max_error_over_tolerance=max_ratio, + failing_elements=failed, + passed=failed == 0, + ) + return result + + +def logical_keys(rows, device): + tokens = torch.arange(rows, device=device, dtype=torch.int64) + return torch.stack((tokens // 257, tokens % 257), dim=1) + + +def training_projection(op, x, weight, grad, keys, *, chunk_size=None): + from rl_engine.kernels.ops.canonical_backward import canonical_backward_session + + x = x.detach().clone().requires_grad_(True) + weight = weight.detach().clone().requires_grad_(True) + traces = [] + with canonical_backward_session() as session: + if chunk_size is None: + output = op(x, weight, logical_keys=keys) + traces.append(op.last_execution) + else: + parts = [] + for lo in range(0, x.shape[0], chunk_size): + parts.append( + op(x[lo : lo + chunk_size], weight, logical_keys=keys[lo : lo + chunk_size]) + ) + traces.append(op.last_execution) + output = torch.cat(parts) + output.backward(grad) + session.validate_complete() + return (output.detach(), x.grad.detach(), weight.grad.detach()), traces diff --git a/rl_engine/kernels/ops/h3_ffn_down.py b/rl_engine/kernels/ops/h3_ffn_down.py new file mode 100644 index 000000000..15c0b513b --- /dev/null +++ b/rl_engine/kernels/ops/h3_ffn_down.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""H3 BF16 down projection: an explicit SM90 reuse candidate for issue #420. + +The default reuses the existing fixed-order FP32 Triton tile kernel. The native +BF16 midpoint tree is retained as an explicit legacy comparison backend. +""" + +from __future__ import annotations + +import torch +from torch.autograd.function import once_differentiable + +from rl_engine.kernels.ops import base +from rl_engine.kernels.ops.backward_runtime import record_backward +from rl_engine.kernels.ops.canonical_backward import active_session + +H3_FFN_DIM = 14336 +H3_HIDDEN_DIM = 5376 +H3_MAX_ROWS = 32768 +H3_DOWN_BACKEND = "rlkernel.h3_ffn_down.triton.fp32_tiles.v1" +H3_LEGACY_DOWN_BACKEND = "rlkernel.h3_ffn_down.sm90.midtree.v1" +H3_CHECKPOINT_REVISION = "42ed227ee7df40d41602854ae760620d6eb651fe" + + +def _validate_inputs(x: torch.Tensor, weight: torch.Tensor) -> None: + for name, value in (("x", x), ("weight", weight)): + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a Tensor") + if value.layout != torch.strided: + raise ValueError(f"{name} must use strided layout") + if value.dtype != torch.bfloat16: + raise TypeError(f"{name} must have dtype bfloat16") + if x.ndim not in (2, 3) or x.shape[-1] != H3_FFN_DIM: + raise ValueError("x must have shape [M,14336] or [B,S,14336]") + if tuple(weight.shape) != (H3_HIDDEN_DIM, H3_FFN_DIM): + raise ValueError("weight must have native shape [5376,14336]") + rows = x.numel() // H3_FFN_DIM + if not 1 <= rows <= H3_MAX_ROWS: + raise ValueError(f"row count must be in [1,{H3_MAX_ROWS}]") + if x.device != weight.device: + raise ValueError("x and weight must share one device") + if x.device.type != "cuda" or torch.version.hip is not None: + raise RuntimeError("h3_ffn_down_gemm requires NVIDIA CUDA") + + +def _require_native(device: torch.device): + if torch.cuda.get_device_capability(device) != (9, 0): + raise RuntimeError("h3_ffn_down_gemm currently supports SM90 only") + extension = base._C + marker = getattr(extension, "det_gemm_sm90_compiled", None) + symbols = ("det_gemm_fwd_rhs_transposed", "det_gemm_fwd", "det_gemm_db_transposed") + if not base._EXT_AVAILABLE or not callable(marker) or not marker(): + raise RuntimeError("rebuild with KERNEL_ALIGN_DET_GEMM_SM90=1; fallback is forbidden") + if any(not callable(getattr(extension, symbol, None)) for symbol in symbols): + raise RuntimeError("native-weight GEMM symbols missing; rebuild the extension") + if torch.backends.cuda.matmul.allow_tf32: + raise RuntimeError("disable TF32 before executing the H3 numerical profile") + return extension + + +def _require_backend(device, backend=H3_DOWN_BACKEND): + if backend == H3_LEGACY_DOWN_BACKEND: + return _require_native(device) + if backend != H3_DOWN_BACKEND: + raise ValueError("unsupported H3 backend") + if torch.cuda.get_device_capability(device) != (9, 0): + raise RuntimeError("h3_ffn_down_gemm currently supports SM90 only") + if torch.backends.cuda.matmul.allow_tf32: + raise RuntimeError("disable TF32 before executing the H3 numerical profile") + from rl_engine.kernels.ops.h3_down_fp32_backend import FixedFP32Backend + + return FixedFP32Backend() + + +def _register_rows(session, weight, keys, rows, parameter_id): + if not isinstance(keys, torch.Tensor) or keys.dtype != torch.int64: + raise TypeError("logical_keys must be an int64 Tensor") + if keys.ndim != 2 or keys.shape[0] != rows or keys.shape[1] not in (2, 3): + raise ValueError("logical_keys must have shape [rows,2] or [rows,3]") + if keys.device != weight.device: + raise ValueError("logical_keys and weight must share one device") + if not isinstance(parameter_id, str) or not parameter_id.strip(): + raise ValueError("parameter_id must be a nonempty stable parameter name") + # Own the keys: caller mutation must not change a pending backward graph. + keys = keys.detach().clone() + bindings = getattr(session, "_h3_down_bindings", None) + if bindings is None: + bindings = {} + session._h3_down_bindings = bindings + previous = bindings.get(parameter_id) + if previous is not None and previous is not weight: + raise ValueError("parameter_id is already bound to another weight Tensor") + if any(value is weight and name != parameter_id for name, value in bindings.items()): + raise ValueError("one weight Tensor must use one parameter_id per session") + entries = session.uses.get(parameter_id, ()) + if any(entry.keys.shape[1] != keys.shape[1] for entry in entries): + raise ValueError("logical key width must remain fixed per parameter") + all_keys = torch.cat([entry.keys for entry in entries] + [keys]) + active = all_keys[all_keys[:, 0] >= 0] + if active.shape[0] > H3_MAX_ROWS: + raise ValueError("canonical active row count exceeds the declared H3 row limit") + if torch.unique(active, dim=0).shape[0] != active.shape[0]: + raise ValueError("active logical keys must be unique across parameter uses") + slot = session.register(parameter_id, keys) + bindings[parameter_id] = weight + return slot, keys + + +def _weight_gradient(extension, rows, grads): + # Fix the token reduction extent from the SAME active logical row set. + # Pad to BK=32 for a stable tail; also bypass legacy scalar/small-K routes. + count = rows.shape[0] + padded = ((count + 31) // 32) * 32 + if padded != count: + x_pad = rows.new_zeros((padded, rows.shape[1])) + g_pad = grads.new_zeros((padded, grads.shape[1])) + x_pad[:count].copy_(rows) + g_pad[:count].copy_(grads) + rows, grads = x_pad, g_pad + return extension.det_gemm_db_transposed(rows.contiguous(), grads.contiguous()) + + +class _H3DownFn(torch.autograd.Function): + @staticmethod + def forward(ctx, x, weight, session, slot, keys, parameter_id, trace): + extension = _require_backend(x.device, trace["requested_backend"]) + flat = x.reshape(-1, H3_FFN_DIM).contiguous() + native_weight = weight.contiguous() + # The native extension uses the current device's stream and SM query. + with torch.cuda.device(x.device): + y = extension.det_gemm_fwd_rhs_transposed(flat, native_weight) + # Retain original version counters even when layout normalization copies. + ctx.save_for_backward(flat, native_weight, keys, x, weight) + ctx.session, ctx.slot, ctx.parameter_id = session, slot, parameter_id + ctx.extension, ctx.trace, ctx.input_shape = extension, trace, x.shape + trace.update(actual_backend=trace["requested_backend"], forward_executed=True) + trace["kernel_id"] = getattr( + extension, "h3_kernel_id", "rl_engine._C.det_gemm_fwd_rhs_transposed" + ) + if hasattr(extension, "h3_last_launch"): + trace["forward_launch"] = extension.h3_last_launch + return y.reshape(*x.shape[:-1], H3_HIDDEN_DIM) + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + x, weight, keys, _original_x, _original_weight = ctx.saved_tensors + grad = grad_output.reshape(-1, H3_HIDDEN_DIM).contiguous() + if grad.dtype != torch.bfloat16: + raise TypeError("H3 upstream gradient must have dtype bfloat16") + if keys is not None and torch.any(grad[keys[:, 0] < 0] != 0).item(): + raise ValueError("padding rows must have zero upstream gradient") + dx = dw = None + executed = [] + with torch.cuda.device(x.device): + if ctx.needs_input_grad[0]: + dx = ctx.extension.det_gemm_fwd(grad, weight).reshape(ctx.input_shape) + executed.append(getattr(ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_fwd")) + ctx.trace["input_gradient_executed"] = True + if hasattr(ctx.extension, "h3_last_launch"): + ctx.trace["input_gradient_launch"] = ctx.extension.h3_last_launch + if ctx.needs_input_grad[1]: + + def reducer(rows, grads): + value = _weight_gradient(ctx.extension, rows, grads) + executed.append( + getattr( + ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_db_transposed" + ) + ) + ctx.trace["weight_gradient_executed"] = True + ctx.trace["canonical_active_rows"] = rows.shape[0] + if hasattr(ctx.extension, "h3_last_launch"): + ctx.trace["weight_gradient_launch"] = ctx.extension.h3_last_launch + return value + + dw = ctx.session.submit_linear(ctx.parameter_id, ctx.slot, x, grad, reducer) + if executed: + record_backward( + "h3_ffn_down_gemm", + kernel_id="+".join(executed), + impl=ctx.trace["requested_backend"], + family="triton" if ctx.trace["requested_backend"] == H3_DOWN_BACKEND else "cuda", + ) + return dx, dw, None, None, None, None, None + + +class H3FFNDownGemmOp: + """Explicit eager CUDA operator; no backend auto-selection or fallback. + + Weight gradients require one canonical_backward_session spanning ALL + forward uses and a single backward call. Supply unique, stable logical keys + [sample_id, token_id] (optionally a third component); first key < 0 is padding. + """ + + def __init__(self, *, backend=H3_DOWN_BACKEND): + if backend not in (H3_DOWN_BACKEND, H3_LEGACY_DOWN_BACKEND): + raise ValueError("unsupported H3 backend") + self.backend = backend + self.last_execution: dict = {} + + def __call__(self, x, weight, *, logical_keys=None, parameter_id="h3.ffn.down.weight"): + _validate_inputs(x, weight) + if torch.compiler.is_compiling(): + raise RuntimeError("this H3 profile currently supports eager execution only") + _require_backend(x.device, self.backend) + session, slot, keys = None, None, None + if torch.is_grad_enabled() and weight.requires_grad: + session = active_session() + if session is None: + raise RuntimeError("weight gradients require an active canonical backward session") + slot, keys = _register_rows( + session, weight, logical_keys, x.numel() // H3_FFN_DIM, parameter_id + ) + trace = { + "requested_backend": self.backend, + "actual_backend": None, + "kernel_id": None, + "fallback": False, + "arithmetic": ( + "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast" + if self.backend == H3_DOWN_BACKEND + else "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes" + ), + "tile": [64, 64, 32] if self.backend == H3_DOWN_BACKEND else [128, 64, 32], + "num_warps": 4, + "num_stages": 2, + "fp_fusion": False if self.backend == H3_DOWN_BACKEND else "native build policy", + "split_k": False, + "split_policy": "none", + "accumulator_dtype": "float32", + "schedule_id": ( + "fixed-64x64x32-w4-s2-fusion0" + if self.backend == H3_DOWN_BACKEND + else "native-midpoint-128x64x32-w4-s2" + ), + "topology": {"world_size": 1, "tensor_parallel_size": 1}, + "checkpoint_revision": H3_CHECKPOINT_REVISION, + "weight_layout": "[5376,14336]", + "dw_order": "lexicographic active logical keys; trailing zero pad to 32", + "forward_executed": False, + "input_gradient_executed": False, + "weight_gradient_executed": False, + "qualification": "pending_h3_gpu_evidence", + } + self.last_execution = trace + return _H3DownFn.apply(x, weight, session, slot, keys, parameter_id, trace) + + +def h3_ffn_down_gemm( + x, weight, *, logical_keys=None, parameter_id="h3.ffn.down.weight", backend=H3_DOWN_BACKEND +): + return H3FFNDownGemmOp(backend=backend)( + x, weight, logical_keys=logical_keys, parameter_id=parameter_id + ) diff --git a/rl_engine/kernels/registry.py b/rl_engine/kernels/registry.py index 9eeb1c42b..32ce9f62e 100644 --- a/rl_engine/kernels/registry.py +++ b/rl_engine/kernels/registry.py @@ -203,6 +203,27 @@ class OpBackend(Enum, metaclass=_KernelEnumMeta): def _default_semantic_descriptors() -> tuple[OperatorBackendDescriptor, ...]: return ( + OperatorBackendDescriptor( + semantic_op="h3_ffn_down_gemm", + backend_id="rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + supported_targets=frozenset({"rollout", "training"}), + supported_devices=frozenset({"cuda"}), + supported_dtypes=frozenset({"bfloat16"}), + supported_topologies={"world_size": (1,), "tensor_parallel_size": (1,)}, + determinism_or_alignment_properties={ + "arithmetic": "fp32_dot_tiles_ordered_fp32_adds_bf16_epilogue", + "shape": "14336_to_5376", + "weight_gradient_order": "canonical_logical_keys", + "gpu_qualified": False, + "strict_observable": True, + }, + lifecycle=OperatorLifecycle.REQUEST, + implementation_class_or_factory=("rl_engine.kernels.ops.h3_ffn_down.H3FFNDownGemmOp"), + fallback_policy=OperatorFallbackPolicy.ERROR, + version_or_build_fingerprint="h3-down-triton-fp32-tiles-candidate-v1", + # Strict production resolution remains gated until H3 GPU evidence exists. + is_smoke_only=True, + ), OperatorBackendDescriptor( semantic_op="selected_logprob", backend_id="rlkernel.reference_logp", diff --git a/scripts/plot_h3_down_report.py b/scripts/plot_h3_down_report.py new file mode 100644 index 000000000..ec0ee9d57 --- /dev/null +++ b/scripts/plot_h3_down_report.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Produce a readable report and static charts from measured H3 evidence only.""" + +import argparse +import csv +import json +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report") + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + report = json.loads(Path(args.report).read_text()) + output = Path(args.output_dir) + output.mkdir(parents=True, exist_ok=True) + cases = report["cases"] + # Render earlier native-midtree reports alongside the versioned FP32 candidate. + for case in cases: + case["benchmarks"] = { + key.replace("native_", "candidate_", 1): value + for key, value in case["benchmarks"].items() + } + lines = [ + "# H3 down projection GPU evidence", + "", + f"Run status: **{report['status']}**.", + "", + "Operator-only evidence; production promotion requires review of the matrix, " + "real checkpoint fixtures and the arithmetic contract.", + "", + ] + if report.get("error"): + lines += ["```text", report["error"], "```", ""] + if report.get("fixture"): + metadata = report["fixture"]["metadata"] + lines += [f"Checkpoint fixture activation kind: **{metadata['activation_kind']}**.", ""] + lines += [ + "| Input | M | Checks passed | Total | Candidate fwd ms | torch BF16 fwd ms |", + "|---|---:|---:|---:|---:|---:|", + ] + for case in cases: + bench = case["benchmarks"] + lines.append( + f"| {case['input_family']} | {case['rows']} | " + f"{sum(row['passed'] for row in case['checks'])} | {len(case['checks'])} | " + f"{bench['candidate_forward']['median_ms']:.4f} | " + f"{bench['torch_bf16_forward']['median_ms']:.4f} |" + ) + lines += ["", "Remaining acceptance:", ""] + lines += [f"- {value}" for value in report.get("remaining_acceptance", [])] + (output / "summary.md").write_text("\n".join(lines) + "\n") + with (output / "checks.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow( + [ + "input", + "M", + "judgment", + "tensor", + "transform", + "passed", + "max_abs_error", + "max_error_over_tolerance", + "mismatched_bytes", + ] + ) + for case in cases: + for row in case["checks"]: + writer.writerow( + [ + case["input_family"], + case["rows"], + row["judgment"], + row["tensor"], + row["transform"], + row["passed"], + row.get("max_abs_error"), + row.get("max_error_over_tolerance"), + row.get("mismatched_bytes"), + ] + ) + if not cases: + return + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + for family in sorted({case["input_family"] for case in cases}): + subset = sorted( + (case for case in cases if case["input_family"] == family), + key=lambda case: case["rows"], + ) + fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) + counts = [case["rows"] for case in subset] + for key, label in ( + ("candidate_forward", "Candidate forward"), + ("torch_bf16_forward", "torch BF16 forward"), + ("candidate_forward_backward", "Candidate forward+backward"), + ("torch_bf16_forward_backward", "torch BF16 forward+backward"), + ): + axes[0].plot( + counts, + [case["benchmarks"][key]["median_ms"] for case in subset], + marker="o", + label=label, + ) + axes[0].set( + xlabel="Logical rows M", + ylabel="Synchronized wall time (ms)", + xscale="log", + yscale="log", + title=family, + ) + axes[0].legend(fontsize=8) + for tensor in ("y", "dx", "dw"): + values = [ + max( + row.get("max_error_over_tolerance", 0) + for row in case["checks"] + if row["tensor"] == tensor and row["transform"] == "fp32_reference" + ) + for case in subset + ] + axes[1].plot(counts, values, marker="o", label=tensor) + axes[1].axhline(1, color="red", linestyle="--", label="shared tolerance boundary") + axes[1].set( + xlabel="Logical rows M", + ylabel="Max error / allowed error", + xscale="log", + title="Accuracy against independent FP32 reference", + ) + axes[1].legend(fontsize=8) + fig.savefig(output / f"{family}.png", dpi=180) + plt.close(fig) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_h3_down_fixture.py b/scripts/prepare_h3_down_fixture.py new file mode 100644 index 000000000..d37d735aa --- /dev/null +++ b/scripts/prepare_h3_down_fixture.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Fetch one pinned H3 down weight by HTTP ranges, with synthetic activations.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import struct +import sys +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import torch # noqa: E402 + +from rl_engine.kernels.ops.h3_ffn_down import H3_CHECKPOINT_REVISION # noqa: E402 + +BASE = f"https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/{H3_CHECKPOINT_REVISION}/" +WEIGHT_KEY = "transformer_blocks.0.ff.net.2.weight" +WEIGHT_SHAPE = [5376, 14336] + + +def read_range(url, start, end, *, opener=urllib.request.urlopen): + """Reject servers that ignore Range instead of downloading a whole shard.""" + request = urllib.request.Request(url, headers={"Range": f"bytes={start}-{end}"}) + with opener(request, timeout=120) as response: + if response.status != 206: + raise RuntimeError(f"expected HTTP 206, got {response.status}; refusing full shard") + match = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", response.headers.get("Content-Range", "")) + if not match or (int(match[1]), int(match[2])) != (start, end): + raise RuntimeError("Content-Range does not match requested tensor bytes") + data = response.read(end - start + 2) + if len(data) != end - start + 1: + raise RuntimeError("range response length mismatch") + return data + + +def tensor_extent(entry, header_size): + if entry["dtype"] != "BF16" or entry["shape"] != WEIGHT_SHAPE: + raise ValueError("checkpoint down weight must be BF16 [5376,14336]") + start, end = entry["data_offsets"] + expected = 5376 * 14336 * 2 + if start < 0 or end - start != expected: + raise ValueError("checkpoint tensor extent does not match its declared shape") + return 8 + header_size + start, 8 + header_size + end + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True) + parser.add_argument("--rows", type=int, default=129) + parser.add_argument("--seed", type=int, default=420) + parser.add_argument( + "--variant", choices=("transformer", "transformer_ref"), default="transformer" + ) + args = parser.parse_args(argv) + if not 1 <= args.rows <= 32768: + parser.error("rows must be in [1,32768]") + index_url = BASE + args.variant + "/diffusion_pytorch_model.safetensors.index.json" + with urllib.request.urlopen(index_url, timeout=60) as response: + index_raw = response.read(8 << 20) + index = json.loads(index_raw) + shard = index["weight_map"][WEIGHT_KEY] + if Path(shard).name != shard or not shard.endswith(".safetensors"): + raise ValueError("unexpected checkpoint shard path") + url = BASE + args.variant + "/" + shard + header_size = struct.unpack("=64 +wheel diff --git a/scripts/run_h3_down_h100.sh b/scripts/run_h3_down_h100.sh new file mode 100644 index 000000000..c9aa2be0e --- /dev/null +++ b/scripts/run_h3_down_h100.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +set -euo pipefail + +repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_dir" +mode="${1:-smoke}" +case "$mode" in smoke|full) ;; *) echo 'Usage: bash scripts/run_h3_down_h100.sh [smoke|full]' >&2; exit 2 ;; esac +result_dir="${H3_RESULT_DIR:-$repo_dir/artifacts/h3-down/$(date -u +%Y%m%dT%H%M%SZ)}" +mkdir -p "$result_dir" +export KERNEL_ALIGN_USE_FAST_MATH=0 +export KERNEL_ALIGN_DET_GEMM_SM90=1 +export KERNEL_ALIGN_FORCE_SM90=0 +export RL_KERNEL_REQUIRE_EXT=1 +export RL_KERNEL_DET_GEMM_BACKEND=sm90 +export MAX_JOBS="${MAX_JOBS:-8}" +export PYTHONUNBUFFERED=1 +# SSH non-login shells in Runpod's development images may omit the toolkit PATH. +if ! command -v nvcc >/dev/null 2>&1 && [[ -x "${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" ]]; then + export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" + export PATH="$CUDA_HOME/bin:$PATH" +fi +# BuildExtension's explicit 90a flags cover the TMA path; do not add plain compute_90 PTX. +unset TORCH_CUDA_ARCH_LIST + +python - <<'PY' +from pathlib import Path +import os +import sysconfig +import torch +if torch.version.hip is not None or not torch.cuda.is_available(): + raise SystemExit('An NVIDIA H100 GPU and a CUDA PyTorch build are required') +if torch.cuda.get_device_capability() != (9, 0): + raise SystemExit('This profile requires SM90') +if torch.version.cuda != '12.4': + raise SystemExit('This initial runbook pins a CUDA 12.4 PyTorch build') +print(torch.__version__, torch.cuda.get_device_name(), torch.cuda.get_device_capability()) +if os.environ.get('H3_BUILD_NATIVE', '0') == '1' and not ( + Path(sysconfig.get_path('include')) / 'Python.h' +).is_file(): + raise SystemExit('Python development headers are required; install the matching pythonX.Y-dev') +PY +nvcc --version | tee "$result_dir/nvcc.txt" +nvcc_version="$(< "$result_dir/nvcc.txt")" +if [[ "$nvcc_version" != *"release 12.4"* ]]; then + echo 'This initial runbook requires nvcc 12.4' >&2 + exit 2 +fi +nvidia-smi -q > "$result_dir/nvidia-smi.txt" +python -m pip freeze > "$result_dir/pip-freeze.txt" +git status --short > "$result_dir/git-status.txt" +git diff HEAD > "$result_dir/source.patch" +python -m pytest -q tests/test_h3_ffn_down.py -k 'not on_gpu' | tee "$result_dir/cpu-tests.txt" +if [[ "${H3_BUILD_NATIVE:-0}" == 1 ]]; then + python -m pip install --no-deps --no-build-isolation -v -e . 2>&1 | tee "$result_dir/build.log" +fi +set +e +python -m pytest -q tests/test_h3_ffn_down.py | tee "$result_dir/gpu-tests.txt" +gpu_test_status=${PIPESTATUS[0]} +set -e + +args=(--output "$result_dir/report.json") +if [[ -f "$result_dir/build.log" ]]; then args+=(--build-log "$result_dir/build.log"); fi +if [[ -n "${H3_BACKEND:-}" ]]; then args+=(--backend "$H3_BACKEND"); fi +if [[ "$mode" == smoke ]]; then + args+=(--rows 1,33,129 --families random --repeats 3) +fi +if [[ -n "${H3_FIXTURE:-}" ]]; then args+=(--fixture "$H3_FIXTURE"); fi +if [[ "${H3_COMPARE_VLLM:-0}" == 1 ]]; then args+=(--compare-vllm); fi +set +e +python scripts/validate_h3_ffn_down.py "${args[@]}" 2>&1 | tee "$result_dir/validation.log" +validation_status=${PIPESTATUS[0]} +set -e +python scripts/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir" +echo "Evidence directory: $result_dir" +if [[ "$gpu_test_status" -ne 0 ]]; then exit "$gpu_test_status"; fi +exit "$validation_status" diff --git a/scripts/validate_h3_ffn_down.py b/scripts/validate_h3_ffn_down.py new file mode 100644 index 000000000..4eb631411 --- /dev/null +++ b/scripts/validate_h3_ffn_down.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Fail-closed H3 GPU qualification with JSON evidence and performance samples.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import statistics +import subprocess +import sys +import time +import traceback +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import torch # noqa: E402 + +from rl_engine.kernels.gtest.tolerance import load_contract # noqa: E402 +from rl_engine.kernels.ops import base # noqa: E402 +from rl_engine.kernels.ops.h3_down_validation import ( # noqa: E402 + comparison, + logical_keys, + reference_projection, + training_projection, +) +from rl_engine.kernels.ops.h3_ffn_down import ( # noqa: E402 + H3_CHECKPOINT_REVISION, + H3_DOWN_BACKEND, + H3_LEGACY_DOWN_BACKEND, + H3_MAX_ROWS, + H3FFNDownGemmOp, + _require_backend, + _weight_gradient, +) + + +def command(args): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=False) + return result.stdout.strip() if result.returncode == 0 else result.stderr.strip() + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def tensor_digest(tensor): + data = tensor.detach().contiguous().view(torch.uint8).cpu().numpy() + return { + "sha256": hashlib.sha256(data).hexdigest(), + "dtype": str(tensor.dtype), + "shape": list(tensor.shape), + } + + +def provenance(args): + contract_path = ROOT / "rl_engine/kernels/gtest/tolerance_contract.json" + result = { + "git_commit": command(["git", "rev-parse", "HEAD"]), + "git_status": command(["git", "status", "--porcelain"]), + "git_diff_sha256": hashlib.sha256(command(["git", "diff", "HEAD"]).encode()).hexdigest(), + "python": platform.python_version(), + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "torch_hip": torch.version.hip, + "contract_version": load_contract()["version"], + "contract_sha256": sha256(contract_path), + "checkpoint_revision": H3_CHECKPOINT_REVISION, + "args": vars(args), + "environment": { + name: os.getenv(name) + for name in ( + "KERNEL_ALIGN_USE_FAST_MATH", + "KERNEL_ALIGN_DET_GEMM_SM90", + "RL_KERNEL_DET_GEMM_BACKEND", + "CUBLAS_WORKSPACE_CONFIG", + "CUBLASLT_WORKSPACE_SIZE", + "TORCH_CUDA_ARCH_LIST", + ) + }, + "source_hashes": {}, + "build_flags_verified_at_runtime": False, + } + for name in ( + "csrc/cuda/gemm/det_gemm_kernel.cu", + "csrc/cuda/gemm/det_gemm_tma.cuh", + "rl_engine/kernels/ops/h3_ffn_down.py", + "rl_engine/kernels/ops/h3_down_fp32_backend.py", + "rl_engine/kernels/ops/triton/matmul/det_gemm.py", + "rl_engine/kernels/ops/h3_down_validation.py", + "rl_engine/kernels/ops/canonical_backward.py", + "rl_engine/kernels/registry.py", + "scripts/validate_h3_ffn_down.py", + ): + result["source_hashes"][name] = sha256(ROOT / name) + if base._EXT_AVAILABLE: + result["extension_sha256"] = sha256(base._C.__file__) + if args.build_log: + result["build_log_sha256"] = sha256(args.build_log) + return result + + +def timed(fn, repeats): + for _ in range(2): + fn() + torch.cuda.synchronize() + base_memory = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + samples = [] + # Wall clock includes host-side canonical-key validation, copies and reduction prep. + for _ in range(repeats): + start = time.perf_counter() + value = fn() + torch.cuda.synchronize() + samples.append((time.perf_counter() - start) * 1000) + del value + return { + "median_ms": statistics.median(samples), + "samples_ms": samples, + "peak_allocated_bytes": torch.cuda.max_memory_allocated(), + "incremental_peak_bytes": torch.cuda.max_memory_allocated() - base_memory, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding", + } + + +def add_checks(report, actual, expected, transform, *, accuracy=False): + for i, (left, right) in enumerate(zip(actual, expected, strict=True)): + judgment = "forward_" if i == 0 else "gradient_" + judgment += "accuracy" if accuracy else "invariance" + row = comparison(left, right, judgment) + row.update(tensor=("y", "dx", "dw")[i], transform=transform) + report["checks"].append(row) + + +def row_local(op, x, w, dy): + x = x.detach().clone().requires_grad_(True) + y = op(x, w.detach()) + (dx,) = torch.autograd.grad(y, x, dy) + return y.detach(), dx.detach() + + +def vllm_comparison(x, w, dy, reference, repeats): + """Optional forward and explicit VJP prior-art probe, without a vLLM install. + + This probe is not an autograd integration or a weight-gradient chunking claim. + """ + result = {"status": "running", "checks": [], "benchmarks": {}} + try: + import triton + + from benchmarks.vllm_batch_invariant_matmul import ( + VLLM_BATCH_INVARIANT_SOURCE_SHA, + matmul_config_metadata, + matmul_persistent, + ) + + result.update( + source_commit=VLLM_BATCH_INVARIANT_SOURCE_SHA, + triton_version=triton.__version__, + source_sha256=sha256(ROOT / "benchmarks/vllm_batch_invariant_matmul.py"), + configs_sha256=sha256(ROOT / "benchmarks/vllm_batch_invariant_configs.py"), + ) + pairs = ((x, w.t()), (dy, w), (dy.t(), x)) + result["configs"] = [ + matmul_config_metadata(a.shape[0], b.shape[1], a.shape[1], a.dtype) for a, b in pairs + ] + actual = tuple(matmul_persistent(a, b) for a, b in pairs) + add_checks(result, actual, reference, "fp32_reference", accuracy=True) + for i, (a, b) in enumerate(pairs): + result["benchmarks"][("forward", "dx", "dw")[i]] = timed( + lambda a=a, b=b: matmul_persistent(a, b), repeats + ) + if i < 2: + chunked = torch.cat( + [matmul_persistent(a[lo : lo + 31], b) for lo in range(0, a.shape[0], 31)] + ) + judgment = "forward_invariance" if i == 0 else "gradient_invariance" + for name, value, expected in ( + ("chunk31", chunked, actual[i]), + ("singleton", matmul_persistent(a[:1], b), actual[i][:1]), + ): + check = comparison(value, expected, judgment) + check.update(tensor=("y", "dx")[i], transform=name) + result["checks"].append(check) + result["status"] = ( + "passed_probe" if all(row["passed"] for row in result["checks"]) else "failed_probe" + ) + result["qualification"] = "partial_prior_art_probe_only" + except Exception: + result["status"] = "error" + result["error"] = traceback.format_exc() + return result + + +def run_case(x, w, dy, label, repeats, *, compare_vllm=False, backend=H3_DOWN_BACKEND): + rows = x.shape[0] + keys = logical_keys(rows, x.device) + op = H3FFNDownGemmOp(backend=backend) + case = {"rows": rows, "input_family": label, "checks": [], "benchmarks": {}} + canonical, traces = training_projection(op, x, w, dy, keys) + case["execution_traces"] = traces + case["output_digests"] = { + name: tensor_digest(value) for name, value in zip(("y", "dx", "dw"), canonical, strict=True) + } + reference = reference_projection(x, w, dy) + add_checks(case, canonical, reference, "fp32_reference", accuracy=True) + if compare_vllm: + case["vllm_comparison"] = vllm_comparison(x, w, dy, reference, repeats) + del reference + repeat, _ = training_projection(op, x, w, dy, keys) + add_checks(case, repeat, canonical, "repeat") + del repeat + order = torch.arange(rows - 1, -1, -1, device=x.device) + inverse = torch.argsort(order) + permuted, _ = training_projection(op, x[order], w, dy[order], keys[order]) + add_checks( + case, + (permuted[0][inverse], permuted[1][inverse], permuted[2]), + canonical, + "permuted_logical_rows", + ) + del permuted + chunked, _ = training_projection(op, x, w, dy, keys, chunk_size=31) + add_checks(case, chunked, canonical, "chunk31_single_backward") + del chunked + # Force non-contiguous activations AND native-shaped non-contiguous weights. + layout_x, layout_w = x.t().contiguous().t(), w.t().contiguous().t() + layout, _ = training_projection(op, layout_x, layout_w, dy, keys) + add_checks(case, layout, canonical, "strided_input_and_weight") + del layout, layout_x, layout_w + pad = 17 + padded_x = torch.cat((x, torch.ones(pad, 14336, device=x.device, dtype=x.dtype))) + padded_dy = torch.cat((dy, torch.zeros(pad, 5376, device=x.device, dtype=dy.dtype))) + padded_keys = torch.cat((keys, torch.full((pad, 2), -1, device=x.device, dtype=torch.int64))) + if rows + pad <= H3_MAX_ROWS: + padded, _ = training_projection(op, padded_x, w, padded_dy, padded_keys) + add_checks( + case, (padded[0][:rows], padded[1][:rows], padded[2]), canonical, "inactive_padding" + ) + del padded + del padded_x, padded_dy, padded_keys + if rows % 2 == 0: + batched, _ = training_projection( + op, x.reshape(2, rows // 2, 14336), w, dy.reshape(2, rows // 2, 5376), keys + ) + add_checks( + case, + (batched[0].reshape(rows, 5376), batched[1].reshape(rows, 14336), batched[2]), + canonical, + "batch2_reshape", + ) + del batched + # Compare only row-local tensors when the effective logical row set changes. + one = row_local(op, x[:1], w, dy[:1]) + mutated_x, mutated_dy = x.clone(), dy.clone() + mutated_x[1:].neg_() + mutated_dy[1:].zero_() + mutated = row_local(op, mutated_x, w, mutated_dy) + for i in (0, 1): + for name, value in ( + ("singleton_vs_batch", canonical[i][:1]), + ("other_rows_mutated", mutated[i][:1]), + ): + check = comparison( + value, one[i], "forward_invariance" if i == 0 else "gradient_invariance" + ) + check.update(tensor=("y", "dx")[i], transform=name) + case["checks"].append(check) + del one, mutated, mutated_x, mutated_dy, canonical + extension = _require_backend(x.device, backend) + case["benchmarks"]["candidate_forward"] = timed(lambda: op(x, w), repeats) + case["benchmarks"]["candidate_dx"] = timed( + lambda: extension.det_gemm_fwd(dy.contiguous(), w.contiguous()), repeats + ) + case["benchmarks"]["candidate_dw_canonical_order"] = timed( + lambda: _weight_gradient(extension, x, dy), repeats + ) + case["benchmarks"]["candidate_forward_backward"] = timed( + lambda: training_projection(op, x, w, dy, keys), repeats + ) + case["benchmarks"]["torch_bf16_forward"] = timed( + lambda: torch.nn.functional.linear(x, w), repeats + ) + + def torch_training(): + tx = x.detach().requires_grad_(True) + tw = w.detach().requires_grad_(True) + ty = torch.nn.functional.linear(tx, tw) + return torch.autograd.grad(ty, (tx, tw), dy) + + case["benchmarks"]["torch_bf16_forward_backward"] = timed(torch_training, repeats) + case["passed"] = all(check["passed"] for check in case["checks"]) + return case + + +def make_inputs(rows, family, seed): + generator = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(rows, 14336, device="cuda", generator=generator) + w = torch.randn(5376, 14336, device="cuda", generator=generator) / 14336**0.5 + if family == "swiglu_like": + up = torch.randn(x.shape, device="cuda", generator=generator) + x = torch.nn.functional.silu(x) * up + elif family == "cancellation": + # Adjacent +/- activation pairs and nearly identical weights create near-zero sums. + x[:, 1::2] = -x[:, ::2] + w[:, 1::2] = w[:, ::2] + 1e-5 + dy = torch.randn(rows, 5376, device="cuda", generator=generator) * 0.1 + return x.bfloat16(), w.bfloat16(), dy.bfloat16() + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", default="1,31,32,33,127,128,129,256,513,1024,2048,4096") + parser.add_argument("--families", default="random,swiglu_like,cancellation") + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--seed", type=int, default=420) + parser.add_argument("--output", required=True) + parser.add_argument("--build-log") + parser.add_argument( + "--backend", choices=(H3_DOWN_BACKEND, H3_LEGACY_DOWN_BACKEND), default=H3_DOWN_BACKEND + ) + parser.add_argument( + "--compare-vllm", + action="store_true", + help="probe the repository's pinned vLLM Triton matrix kernel", + ) + parser.add_argument( + "--fixture", help="local weights_only torch file with x/weight/grad_output/metadata" + ) + args = parser.parse_args(argv) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + report = { + "schema": "h3-down-qualification-v2", + "backend": args.backend, + "status": "running", + "cases": [], + "acceptance_complete": False, + "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", + } + exit_code = 2 + + def save(): + output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + + try: + report["provenance"] = provenance(args) + if not torch.cuda.is_available() or torch.version.hip is not None: + raise RuntimeError("an NVIDIA SM90 GPU is required; CPU runs are not GPU acceptance") + rows = [int(value) for value in args.rows.split(",")] + families = args.families.split(",") + if not rows or any(not 1 <= value <= H3_MAX_ROWS for value in rows): + raise ValueError("rows must be in [1,32768]") + if args.repeats <= 0 or not set(families) <= {"random", "swiglu_like", "cancellation"}: + raise ValueError("invalid repeats or input family") + torch.set_float32_matmul_precision("highest") + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False + report["provenance"].update( + device=torch.cuda.get_device_name(), + capability=list(torch.cuda.get_device_capability()), + candidate_tf32_enabled=torch.backends.cuda.matmul.allow_tf32, + reference_tf32_enabled=False, + torch_baseline_bf16_reduced_precision_reduction=( + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction + ), + nvidia_smi=command( + [ + "nvidia-smi", + "--query-gpu=name,uuid,driver_version,memory.total", + "--format=csv,noheader", + ] + ), + ) + _require_backend(torch.device("cuda"), args.backend) + save() + for family in families: + for count in rows: + values = make_inputs(count, family, args.seed) + case = run_case( + *values, + family, + args.repeats, + compare_vllm=args.compare_vllm, + backend=args.backend, + ) + report["cases"].append(case) + save() + print(f"{family} M={count}: {'PASS' if case['passed'] else 'FAIL'}", flush=True) + del values, case + if args.fixture: + fixture = torch.load(args.fixture, map_location="cpu", weights_only=True) + metadata = fixture["metadata"] + if metadata.get("checkpoint_revision") != H3_CHECKPOINT_REVISION: + raise ValueError("fixture checkpoint revision does not match the H3 contract") + if not metadata.get("weight_key") or not metadata.get("activation_source"): + raise ValueError("fixture must declare weight_key and activation_source") + if metadata.get("activation_kind") not in {"captured", "synthetic"}: + raise ValueError("fixture activation_kind must be captured or synthetic") + # Metadata must remain serializable even on a later validation failure. + json.dumps(metadata, allow_nan=False) + report["fixture"] = {"sha256": sha256(args.fixture), "metadata": metadata} + values = [fixture[key].to("cuda") for key in ("x", "weight", "grad_output")] + if any(value.dtype != torch.bfloat16 for value in values): + raise TypeError("fixture tensors must already be quantized BF16") + values[0] = values[0].reshape(-1, 14336) + values[2] = values[2].reshape(-1, 5376) + report["cases"].append( + run_case( + *values, + "checkpoint_fixture", + args.repeats, + compare_vllm=args.compare_vllm, + backend=args.backend, + ) + ) + passed = all(case["passed"] for case in report["cases"]) + report["status"] = "passed" if passed else "failed" + report["acceptance_complete"] = False + report["remaining_acceptance"] = [ + "maintainer review of arithmetic contract and covered matrix", + "independent build/environment repeat and prior-art performance decision", + ] + if not args.fixture: + report["remaining_acceptance"].append("real checkpoint weights and captured inputs") + elif report["fixture"]["metadata"]["activation_kind"] != "captured": + report["remaining_acceptance"].append("real captured SwiGLU activations") + if not args.build_log and args.backend == H3_LEGACY_DOWN_BACKEND: + report["remaining_acceptance"].append("auditable no-fast-math extension build log") + exit_code = 0 if passed else 1 + except Exception: + report["status"] = "error" + report["error"] = traceback.format_exc() + print(report["error"], file=sys.stderr) + finally: + save() + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_h3_down_fixture.py b/tests/test_h3_down_fixture.py new file mode 100644 index 000000000..116861f42 --- /dev/null +++ b/tests/test_h3_down_fixture.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Range-download safety checks, without checkpoint network calls.""" + +import importlib.util +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location( + "h3_fixture_under_test", + Path(__file__).resolve().parents[1] / "scripts/prepare_h3_down_fixture.py", +) +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) + + +class Response: + def __init__(self, status, content_range, data): + self.status = status + self.headers = {"Content-Range": content_range} + self.data = data + self.read_calls = [] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def read(self, count): + self.read_calls.append(count) + return self.data[:count] + + +def test_range_download_limits_read_and_preserves_requested_bytes(): + response = Response(206, "bytes 10-13/100", b"abcd") + + def opener(request, timeout): + assert request.get_header("Range") == "bytes=10-13" + assert timeout == 120 + return response + + assert fixture.read_range("https://example.test/shard", 10, 13, opener=opener) == b"abcd" + assert response.read_calls == [5] + + +@pytest.mark.parametrize( + "status,content_range,data,error", + [ + (200, "", b"full shard", "refusing full shard"), + (206, "bytes 0-3/100", b"abcd", "Content-Range"), + (206, "bytes 10-13/100", b"abc", "length mismatch"), + (206, "bytes 10-13/100", b"abcde", "length mismatch"), + ], +) +def test_range_download_rejects_invalid_ranges(status, content_range, data, error): + response = Response(status, content_range, data) + with pytest.raises(RuntimeError, match=error): + fixture.read_range("https://example.test/shard", 10, 13, opener=lambda *a, **k: response) + if status == 200: + assert response.read_calls == [] + + +def test_tensor_extent_rejects_wrong_shape_dtype_or_byte_count(): + entry = {"dtype": "BF16", "shape": [5376, 14336], "data_offsets": [1024, 154141696]} + assert fixture.tensor_extent(entry, 512) == (1544, 154142216) + for changed in ({"dtype": "F32"}, {"shape": [14336, 5376]}, {"data_offsets": [0, 2]}): + with pytest.raises(ValueError): + fixture.tensor_extent(entry | changed, 512) diff --git a/tests/test_h3_down_tooling.py b/tests/test_h3_down_tooling.py new file mode 100644 index 000000000..7c7fece91 --- /dev/null +++ b/tests/test_h3_down_tooling.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""CLI error evidence and report rendering checks without a GPU.""" + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_gpu_validator_refuses_cpu_and_preserves_error_evidence(tmp_path, monkeypatch): + import torch + + spec = importlib.util.spec_from_file_location( + "h3_validator_under_test", ROOT / "scripts/validate_h3_ffn_down.py" + ) + validator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(validator) + # Exercise the CPU branch without reinitializing this host's WSL CUDA driver. + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + report_path = tmp_path / "report.json" + status = validator.main( + [ + "--rows", + "1", + "--families", + "random", + "--output", + str(report_path), + ], + ) + assert status == 2 + report = json.loads(report_path.read_text()) + assert report["status"] == "error" + assert not report["acceptance_complete"] + assert not report["cases"] + assert "SM90 GPU is required" in report["error"] + assert report["provenance"]["contract_version"] == "ws1-c1-v2" + assert report["provenance"]["source_hashes"] + + +def test_failed_run_renders_summary_without_inventing_charts(tmp_path): + source = tmp_path / "report.json" + source.write_text(json.dumps({"status": "error", "cases": [], "error": "no SM90 GPU"})) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/plot_h3_down_report.py"), + str(source), + "--output-dir", + str(tmp_path), + ], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "**error**" in (tmp_path / "summary.md").read_text() + assert "no SM90 GPU" in (tmp_path / "summary.md").read_text() + assert list(tmp_path.glob("*.png")) == [] + + +def test_plot_pipeline_with_explicitly_synthetic_test_data(tmp_path): + checks = [ + { + "judgment": "forward_accuracy" if name == "y" else "gradient_accuracy", + "tensor": name, + "transform": "fp32_reference", + "passed": True, + "max_error_over_tolerance": 0.5, + "max_abs_error": 0.01, + } + for name in ("y", "dx", "dw") + ] + keys = ( + "candidate_forward", + "candidate_forward_backward", + "torch_bf16_forward", + "torch_bf16_forward_backward", + ) + source = tmp_path / "synthetic-test-report.json" + source.write_text( + json.dumps( + { + "status": "test_fixture", + "cases": [ + { + "rows": 1, + "input_family": "synthetic_test_data", + "checks": checks, + "benchmarks": {key: {"median_ms": 1.0} for key in keys}, + } + ], + } + ) + ) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/plot_h3_down_report.py"), + str(source), + "--output-dir", + str(tmp_path), + ], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert (tmp_path / "synthetic_test_data.png").read_bytes().startswith(b"\x89PNG") + assert "synthetic_test_data" in (tmp_path / "checks.csv").read_text() diff --git a/tests/test_h3_ffn_down.py b/tests/test_h3_ffn_down.py new file mode 100644 index 000000000..23ae1ba99 --- /dev/null +++ b/tests/test_h3_ffn_down.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""CPU contract/orchestration checks and separate full-width SM90 checks.""" + +from contextlib import nullcontext + +import pytest +import torch + +import rl_engine.kernels.ops.h3_ffn_down as down +from rl_engine.kernels.ops.canonical_backward import canonical_backward_session +from rl_engine.kernels.ops.h3_down_validation import ( + comparison, + logical_keys, + raw_equal, + reference_projection, + training_projection, +) + + +def test_dimensions_and_checkpoint_are_pinned(): + assert (down.H3_FFN_DIM, down.H3_HIDDEN_DIM, down.H3_MAX_ROWS) == (14336, 5376, 32768) + assert down.H3_CHECKPOINT_REVISION == "42ed227ee7df40d41602854ae760620d6eb651fe" + + +@pytest.mark.parametrize("shape", [(14336,), (2, 7), (1, 1, 1, 14336), (0, 14336), (32769, 14336)]) +def test_rejects_unsupported_shapes_before_dispatch(shape): + with pytest.raises(ValueError): + down.h3_ffn_down_gemm( + torch.empty(shape, dtype=torch.bfloat16, device="meta"), + torch.empty((5376, 14336), dtype=torch.bfloat16, device="meta"), + ) + + +def test_native_weight_layout_and_dtype_are_required(): + x = torch.empty((1, 14336), dtype=torch.bfloat16, device="meta") + with pytest.raises(ValueError, match="native shape"): + down.h3_ffn_down_gemm(x, torch.empty((14336, 5376), dtype=torch.bfloat16, device="meta")) + with pytest.raises(TypeError, match="bfloat16"): + down.h3_ffn_down_gemm( + x.float(), torch.empty((5376, 14336), dtype=torch.bfloat16, device="meta") + ) + + +def test_cpu_cannot_silently_execute_cuda_profile(): + with pytest.raises(RuntimeError, match="NVIDIA CUDA"): + down.h3_ffn_down_gemm( + torch.empty((1, 14336), dtype=torch.bfloat16), + torch.empty((5376, 14336), dtype=torch.bfloat16), + ) + + +def test_missing_extension_and_wrong_hardware_fail_closed(monkeypatch): + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 0)) + with pytest.raises(RuntimeError, match="SM90"): + down._require_native(torch.device("cuda")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + monkeypatch.setattr(down.base, "_EXT_AVAILABLE", False) + with pytest.raises(RuntimeError, match="fallback is forbidden"): + down._require_native(torch.device("cuda")) + + +def test_fp32_backend_missing_triton_and_wrong_hardware_fail_closed(monkeypatch): + from rl_engine.kernels.ops.triton.matmul import det_gemm + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 0)) + with pytest.raises(RuntimeError, match="SM90"): + down._require_backend(torch.device("cuda")) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) + monkeypatch.setattr(det_gemm, "_TRITON_AVAILABLE", False) + with pytest.raises(RuntimeError, match="fallback is forbidden"): + down._require_backend(torch.device("cuda")) + + +def test_raw_bytes_and_nonfinite_policy(): + positive = torch.tensor([0.0], dtype=torch.bfloat16) + negative = torch.tensor([-0.0], dtype=torch.bfloat16) + assert torch.equal(positive, negative) + assert not raw_equal(positive, negative) + assert not comparison(positive, negative, "forward_invariance")["passed"] + nan = torch.tensor([float("nan")], dtype=torch.bfloat16) + assert raw_equal(nan, nan) + assert not comparison(nan, nan, "forward_invariance")["passed"] + + +def test_fp32_reference_keeps_quantized_inputs_and_gradients(): + x = torch.tensor([[0.25, -0.5], [1.0, 0.125]], dtype=torch.bfloat16) + w = torch.tensor([[0.5, 2.0]], dtype=torch.bfloat16) + dy = torch.tensor([[0.5], [-0.25]], dtype=torch.bfloat16) + y, dx, dw = reference_projection(x, w, dy) + assert y.dtype == dx.dtype == dw.dtype == torch.float32 + torch.testing.assert_close(y.double(), x.double() @ w.double().t(), atol=0, rtol=0) + torch.testing.assert_close(dx.double(), dy.double() @ w.double(), atol=0, rtol=0) + torch.testing.assert_close(dw.double(), dy.double().t() @ x.double(), atol=0, rtol=0) + + +class _CPUOrchestrationOnlyExtension: + """Surrogate for testing session flow; does NOT emulate CUDA arithmetic.""" + + def __init__(self): + self.weight_reduction_inputs = [] + + def det_gemm_fwd_rhs_transposed(self, x, w): + return (x.float() @ w.float().t()).bfloat16() + + def det_gemm_fwd(self, x, w): + return (x.float() @ w.float()).bfloat16() + + def det_gemm_db_transposed(self, x, dy): + self.weight_reduction_inputs.append((x.clone(), dy.clone())) + return (dy.float().t() @ x.float()).bfloat16() + + +@pytest.fixture +def cpu_orchestration(monkeypatch): + extension = _CPUOrchestrationOnlyExtension() + monkeypatch.setattr(down, "H3_FFN_DIM", 4) + monkeypatch.setattr(down, "H3_HIDDEN_DIM", 3) + monkeypatch.setattr(down, "_validate_inputs", lambda x, w: None) + monkeypatch.setattr(down, "_require_backend", lambda device, backend: extension) + monkeypatch.setattr(torch.cuda, "device", lambda device: nullcontext()) + return extension + + +def test_canonical_chunk_permutation_and_padding_orchestration(cpu_orchestration): + generator = torch.Generator().manual_seed(42) + x = torch.randn(7, 4, generator=generator).bfloat16() + w = torch.randn(3, 4, generator=generator).bfloat16() + dy = torch.randn(7, 3, generator=generator).bfloat16() + keys = logical_keys(7, "cpu") + op = down.H3FFNDownGemmOp() + canonical, _ = training_projection(op, x, w, dy, keys) + chunked, traces = training_projection(op, x, w, dy, keys, chunk_size=2) + assert all(raw_equal(a, b) for a, b in zip(canonical, chunked, strict=True)) + assert sum(trace["weight_gradient_executed"] for trace in traces) == 1 + order = torch.tensor([6, 1, 3, 0, 5, 2, 4]) + permuted, _ = training_projection(op, x[order], w, dy[order], keys[order]) + assert raw_equal(canonical[2], permuted[2]) + torch.testing.assert_close(permuted[0], canonical[0][order], atol=0, rtol=0) + padded_x = torch.cat((x, torch.ones(2, 4).bfloat16())) + padded_dy = torch.cat((dy, torch.zeros(2, 3).bfloat16())) + padded_keys = torch.cat((keys, torch.full((2, 2), -1, dtype=torch.int64))) + padded, _ = training_projection(op, padded_x, w, padded_dy, padded_keys) + assert raw_equal(padded[2], canonical[2]) + for rows, grads in cpu_orchestration.weight_reduction_inputs: + assert rows.shape[0] == grads.shape[0] == 32 + assert raw_equal(rows[:7], x) + assert torch.count_nonzero(rows[7:]) == torch.count_nonzero(grads[7:]) == 0 + + +def test_training_requires_session_and_unique_stable_keys(cpu_orchestration): + x, w = torch.ones(2, 4).bfloat16(), torch.ones(3, 4).bfloat16().requires_grad_(True) + op = down.H3FFNDownGemmOp() + with pytest.raises(RuntimeError, match="active canonical"): + op(x, w) + with canonical_backward_session(): + keys = logical_keys(2, "cpu") + op(x, w, logical_keys=keys) + with pytest.raises(ValueError, match="unique"): + op(x, w, logical_keys=keys) + with pytest.raises(ValueError, match="another weight"): + op(x, w.detach().clone().requires_grad_(True), logical_keys=keys + 10) + with pytest.raises(ValueError, match="one parameter_id"): + op(x, w, logical_keys=keys + 10, parameter_id="other") + + +def test_keys_are_snapshotted_and_incomplete_backward_is_detected(cpu_orchestration): + x = torch.ones(2, 4).bfloat16().requires_grad_(True) + w = torch.ones(3, 4).bfloat16().requires_grad_(True) + keys = logical_keys(2, "cpu") + with canonical_backward_session() as session: + y = down.h3_ffn_down_gemm(x, w, logical_keys=keys) + keys.fill_(-1) + with pytest.raises(RuntimeError, match="incomplete"): + session.validate_complete() + y.backward(torch.ones_like(y)) + session.validate_complete() + assert w.grad is not None + + +def test_padding_only_first_chunk_and_3d_gradient_shape(cpu_orchestration): + x = torch.ones(2, 4).bfloat16().requires_grad_(True) + w = torch.ones(3, 4).bfloat16().requires_grad_(True) + op = down.H3FFNDownGemmOp() + with canonical_backward_session() as session: + padding = op(x[:1], w, logical_keys=torch.full((1, 2), -1, dtype=torch.int64)) + active = op(x[1:].reshape(1, 1, 4), w, logical_keys=logical_keys(1, "cpu")) + torch.autograd.backward( + (padding, active), (torch.zeros_like(padding), torch.ones_like(active)) + ) + session.validate_complete() + assert x.grad.shape == (2, 4) + assert torch.count_nonzero(x.grad[0]) == 0 + assert raw_equal(w.grad, torch.ones_like(w)) + + +@pytest.mark.parametrize("mutated", ["x", "weight"]) +def test_strided_inputs_keep_autograd_version_checks(cpu_orchestration, mutated): + x = torch.ones(4, 2).bfloat16().t().requires_grad_(True) + w = torch.ones(4, 3).bfloat16().t().requires_grad_(True) + assert not x.is_contiguous() and not w.is_contiguous() + with canonical_backward_session(): + y = down.h3_ffn_down_gemm(x, w, logical_keys=logical_keys(2, "cpu")) + with torch.no_grad(): + (x if mutated == "x" else w).add_(1) + with pytest.raises(RuntimeError, match="modified by an inplace"): + y.backward(torch.ones_like(y)) + + +def test_candidate_registration_cannot_enter_default_strict_resolution(): + from rl_engine.kernels.registry import _default_semantic_descriptors + from rl_engine.kernels.semantic_registry import ( + OperatorRequirements, + OperatorResolutionError, + OperatorResolutionPolicy, + SemanticOperatorCatalog, + ) + + catalog = SemanticOperatorCatalog(_default_semantic_descriptors()) + request = dict( + semantic_op="h3_ffn_down_gemm", + requested_backend=down.H3_DOWN_BACKEND, + target="training", + requirements=OperatorRequirements( + device="cuda", dtype="bfloat16", topology={"world_size": 1, "tensor_parallel_size": 1} + ), + ) + with pytest.raises(OperatorResolutionError): + catalog.session().resolve(**request) + session = catalog.session(OperatorResolutionPolicy(allow_test_backends=True)) + resolved = session.resolve(**request) + instance = session.instantiate(resolved) + assert isinstance(instance, down.H3FFNDownGemmOp) + assert instance.last_execution == {} + assert not resolved.descriptor.determinism_or_alignment_properties["gpu_qualified"] + + +def test_nonzero_padding_gradient_is_rejected(cpu_orchestration): + x, w = torch.ones(2, 4).bfloat16(), torch.ones(3, 4).bfloat16().requires_grad_(True) + keys = torch.tensor([[0, 0], [-1, -1]]) + with canonical_backward_session(): + y = down.h3_ffn_down_gemm(x, w, logical_keys=keys) + with pytest.raises(ValueError, match="zero upstream"): + y.backward(torch.ones_like(y)) + + +requires_sm90 = pytest.mark.skipif( + not torch.cuda.is_available() + or torch.version.hip is not None + or torch.cuda.get_device_capability() != (9, 0), + reason="requires SM90; a missing requested backend on SM90 is a failure", +) + + +@pytest.mark.cuda_only +@requires_sm90 +@pytest.mark.parametrize("rows", [1, 33, 129]) +def test_full_h3_accuracy_and_four_judgments_on_gpu(rows): + torch.backends.cuda.matmul.allow_tf32 = False + generator = torch.Generator(device="cuda").manual_seed(420) + x = torch.randn(rows, 14336, device="cuda", generator=generator).bfloat16() + w = (torch.randn(5376, 14336, device="cuda", generator=generator) / 14336**0.5).bfloat16() + dy = (torch.randn(rows, 5376, device="cuda", generator=generator) * 0.1).bfloat16() + keys = logical_keys(rows, "cuda") + op = down.H3FFNDownGemmOp() + actual, _ = training_projection(op, x, w, dy, keys) + expected = reference_projection(x, w, dy) + for index, (left, right) in enumerate(zip(actual, expected, strict=True)): + judgment = "forward_accuracy" if index == 0 else "gradient_accuracy" + result = comparison(left, right, judgment) + assert result["passed"], result + chunked, _ = training_projection(op, x, w, dy, keys, chunk_size=31) + for index, (left, right) in enumerate(zip(chunked, actual, strict=True)): + judgment = "forward_invariance" if index == 0 else "gradient_invariance" + result = comparison(left, right, judgment) + assert result["passed"], result From b07ab3cb97bfee980f379e9843f5381d0f6db488 Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:10:35 +0800 Subject: [PATCH 02/10] Make H3 validation setup comment platform neutral Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- scripts/run_h3_down_h100.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_h3_down_h100.sh b/scripts/run_h3_down_h100.sh index 29854dbb9..90193f402 100644 --- a/scripts/run_h3_down_h100.sh +++ b/scripts/run_h3_down_h100.sh @@ -16,7 +16,7 @@ export RL_KERNEL_REQUIRE_EXT=1 export RL_KERNEL_DET_GEMM_BACKEND=sm90 export MAX_JOBS="${MAX_JOBS:-8}" export PYTHONUNBUFFERED=1 -# SSH non-login shells in Runpod's development images may omit the toolkit PATH. +# Non-login development shells may omit the CUDA toolkit from PATH. if ! command -v nvcc >/dev/null 2>&1 && [[ -x "${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" ]]; then export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" export PATH="$CUDA_HOME/bin:$PATH" From 2f3126845a02cb02d782cad6781a9bfa1c91901f Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:24:39 +0800 Subject: [PATCH 03/10] Condense H3 down projection evidence and preserve historical provenance Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- .../evidence/h3-down-h100-20261010/SHA256SUMS | 5 - .../h100-cancellation.json | 10459 --------------- .../h3-down-h100-20261010/h100-random.json | 10459 --------------- .../h3-down-h100-20261010/h100-replays.json | 11164 ---------------- .../h3-down-h100-20261010/h100-summary.json | 3112 ----- .../h100-swiglu-like.json | 10459 --------------- docs/operators/h3-ffn-down-results.json | 968 ++ docs/operators/h3-ffn-down-results.sha256 | 1 + docs/operators/h3-ffn-down.md | 75 +- 9 files changed, 991 insertions(+), 45711 deletions(-) delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-random.json delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-replays.json delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-summary.json delete mode 100644 docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json create mode 100644 docs/operators/h3-ffn-down-results.json create mode 100644 docs/operators/h3-ffn-down-results.sha256 diff --git a/docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS b/docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS deleted file mode 100644 index 2787a3c2a..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS +++ /dev/null @@ -1,5 +0,0 @@ -2088317c29cc9de80540f215f54b7d6d660044a568c1f50f6194449b2c9b4fba h100-cancellation.json -66fcd857d26a57ea03ce0984f8bf20db1785336c61d302dce74dfd6a4cf213fb h100-random.json -ee532d0a128eed4e978070fcc9fc6358c0cc0b72eafab50266c4645927cb438c h100-replays.json -3a44e25a9397373010b75caee1ee321864e960fc83594772497f95bfeab15786 h100-summary.json -dcc64cf4360bbb9dca0211319f635cffed1cd91ebc6ddfd24052882a7201ba9d h100-swiglu-like.json diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json b/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json deleted file mode 100644 index d35826eb0..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json +++ /dev/null @@ -1,10459 +0,0 @@ -{ - "schema_version": 1, - "gpu": "h100", - "provenance": { - "derivative": "Sanitized historical evidence; only execution paths replaced with portable placeholders. All measurements and findings retained. No new GPU run.", - "original_commit": "006021ca9875cdc4ae66af7ed184514070f3274a", - "original_path": "docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json" - }, - "artifacts": { - "h100-cancellation.json": { - "original_sha256": "608483ed60de33c388e4bb7bc090fac71619a0d31deceefdac12ea099a6fcb09", - "format": "json", - "content": { - "schema": "h3-down-measured-case-part-v1", - "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", - "case_indexes": [ - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47 - ], - "cases": [ - { - "rows": 1, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.52587890625e-05, - "max_error_over_tolerance": 0.00030462356517091393, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0004891455173492432, - "max_error_over_tolerance": 0.0047654458321630955, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.03244646266102791, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5236491560935974, - "samples_ms": [ - 0.5236491560935974, - 0.5030222237110138, - 0.5003213882446289, - 0.5344152450561523, - 0.7693693041801453 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.21550804376602173, - "samples_ms": [ - 0.5729421973228455, - 0.3075897693634033, - 0.21550804376602173, - 0.21023675799369812, - 0.21257996559143066 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.29595568776130676, - "samples_ms": [ - 0.4697144031524658, - 0.29595568776130676, - 0.2908147871494293, - 0.3237724304199219, - 0.23474544286727905 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.9405077695846558, - "samples_ms": [ - 4.387270659208298, - 4.102259874343872, - 3.798823803663254, - 3.9236098527908325, - 3.9405077695846558 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08775666356086731, - "samples_ms": [ - 0.10928884148597717, - 0.09210780262947083, - 0.08775666356086731, - 0.08741766214370728, - 0.0855475664138794 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.7773749530315399, - "samples_ms": [ - 1.0754838585853577, - 0.7773749530315399, - 0.8160248398780823, - 0.6982982158660889, - 0.47647953033447266 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 31, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5269499272108078e-05, - "max_error_over_tolerance": 0.00030487452750094235, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009725391864776611, - "max_error_over_tolerance": 0.009215985424816608, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007811069488525391, - "max_error_over_tolerance": 0.05571417137980461, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5122534930706024, - "samples_ms": [ - 0.553864985704422, - 0.5489625036716461, - 0.5122534930706024, - 0.4660859704017639, - 0.45041367411613464 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19108504056930542, - "samples_ms": [ - 0.19708648324012756, - 0.19152089953422546, - 0.19108504056930542, - 0.18773600459098816, - 0.18952414393424988 - ], - "peak_allocated_bytes": 224250880, - "incremental_peak_bytes": 888832, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.16224011778831482, - "samples_ms": [ - 0.17355382442474365, - 0.16224011778831482, - 0.15947222709655762, - 0.15872344374656677, - 0.16421452164649963 - ], - "peak_allocated_bytes": 378764288, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.283212125301361, - "samples_ms": [ - 4.283212125301361, - 4.39327210187912, - 4.2417012155056, - 4.258990287780762, - 4.3707191944122314 - ], - "peak_allocated_bytes": 537462784, - "incremental_peak_bytes": 314100736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09158998727798462, - "samples_ms": [ - 0.10312721133232117, - 0.09158998727798462, - 0.08689984679222107, - 0.08834898471832275, - 0.0958070158958435 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2784041464328766, - "samples_ms": [ - 1.018371433019638, - 1.3990290462970734, - 1.0924860835075378, - 1.2784041464328766, - 1.558348536491394 - ], - "peak_allocated_bytes": 378724864, - "incremental_peak_bytes": 155362816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 31, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "dceb60bc57d87aedd2f14d50ca48ec4f6bf5ebd8620a62556cc66e83edf007bc", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 5376 - ] - }, - "dx": { - "sha256": "1b6fbf79020c8c439afedaf519fbbc705df1f51d1cdd7bdb9350d3bae86e7e9a", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 14336 - ] - }, - "dw": { - "sha256": "cdb1d812e7bed96785862d814fae7f6331a1dfcbb93b80d71ee4d48910dff433", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5269499272108078e-05, - "max_error_over_tolerance": 0.00030487452750094235, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009725391864776611, - "max_error_over_tolerance": 0.009215985424816608, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.05573455989360809, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5172304809093475, - "samples_ms": [ - 0.5358867347240448, - 0.5300790071487427, - 0.5172304809093475, - 0.49916282296180725, - 0.5000084638595581 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.22250041365623474, - "samples_ms": [ - 0.24210289120674133, - 0.22250041365623474, - 0.22366270422935486, - 0.2074204385280609, - 0.20667538046836853 - ], - "peak_allocated_bytes": 224347648, - "incremental_peak_bytes": 917504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.15189871191978455, - "samples_ms": [ - 0.16543269157409668, - 0.15189871191978455, - 0.14640763401985168, - 0.1889653503894806, - 0.1428835093975067 - ], - "peak_allocated_bytes": 377570816, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.7806369364261627, - "samples_ms": [ - 3.911960870027542, - 3.7806369364261627, - 3.7540942430496216, - 3.8062632083892822, - 3.6593489348888397 - ], - "peak_allocated_bytes": 536416256, - "incremental_peak_bytes": 312986112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08858740329742432, - "samples_ms": [ - 0.10128691792488098, - 0.09026005864143372, - 0.08858740329742432, - 0.08736178278923035, - 0.0874735414981842 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8712038397789001, - "samples_ms": [ - 0.9313859045505524, - 0.830329954624176, - 0.9034276008605957, - 0.8625313639640808, - 0.8712038397789001 - ], - "peak_allocated_bytes": 378832384, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 32, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "97a7cecd5fcabd2832fe2a7f66e17a7a18bfa2ea038e74ba17d39a8b5faf750d", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 5376 - ] - }, - "dx": { - "sha256": "839af44e35ad707cd7d78d8d1c84034cc6caaf890d04abd959025a9e2c9197ea", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 14336 - ] - }, - "dw": { - "sha256": "4285d8027692d41f18b2c737bf635a5371d268a17b44ea6e93f5cb196ac945cd", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.527508720755577e-05, - "max_error_over_tolerance": 0.00030498611158691347, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000972449779510498, - "max_error_over_tolerance": 0.009215136989951134, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.055741358548402786, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.45315176248550415, - "samples_ms": [ - 0.4736557602882385, - 0.45315176248550415, - 0.45205652713775635, - 0.4499666392803192, - 0.4560798406600952 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.18961727619171143, - "samples_ms": [ - 0.20431727170944214, - 0.19268319010734558, - 0.18948689103126526, - 0.18567591905593872, - 0.18961727619171143 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.1841410994529724, - "samples_ms": [ - 0.2075396478176117, - 0.19084662199020386, - 0.1840628683567047, - 0.1841410994529724, - 0.17883256077766418 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.174433648586273, - "samples_ms": [ - 4.592347890138626, - 4.004254937171936, - 4.059191793203354, - 4.174433648586273, - 4.196617752313614 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08811056613922119, - "samples_ms": [ - 0.10736286640167236, - 0.09493529796600342, - 0.08811056613922119, - 0.08622929453849792, - 0.08698925375938416 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8363053202629089, - "samples_ms": [ - 0.872097909450531, - 0.9182840585708618, - 0.81629678606987, - 0.773504376411438, - 0.8363053202629089 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 127, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5271827578544617e-05, - "max_error_over_tolerance": 0.0003049470251426101, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000971376895904541, - "max_error_over_tolerance": 0.009242621250450611, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.08660231530666351, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.517033040523529, - "samples_ms": [ - 0.517033040523529, - 0.569421797990799, - 0.5456507205963135, - 0.4870966076850891, - 0.45920535922050476 - ], - "peak_allocated_bytes": 231267328, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19916146993637085, - "samples_ms": [ - 0.2094767987728119, - 0.20661205053329468, - 0.19916146993637085, - 0.19781291484832764, - 0.1978762447834015 - ], - "peak_allocated_bytes": 233998336, - "incremental_peak_bytes": 4096512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2178773283958435, - "samples_ms": [ - 0.2397596836090088, - 0.23138150572776794, - 0.2178773283958435, - 0.21693482995033264, - 0.21483004093170166 - ], - "peak_allocated_bytes": 389515264, - "incremental_peak_bytes": 159613440, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.137318581342697, - "samples_ms": [ - 3.5342201590538025, - 3.763865679502487, - 4.137318581342697, - 4.187077283859253, - 4.1816383600234985 - ], - "peak_allocated_bytes": 563264512, - "incremental_peak_bytes": 333362688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09280070662498474, - "samples_ms": [ - 0.10374188423156738, - 0.09563937783241272, - 0.09280070662498474, - 0.09193271398544312, - 0.09140744805335999 - ], - "peak_allocated_bytes": 231267328, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8912608027458191, - "samples_ms": [ - 1.2561529874801636, - 1.1298805475234985, - 0.8912608027458191, - 0.8488893508911133, - 0.7098987698554993 - ], - "peak_allocated_bytes": 389049344, - "incremental_peak_bytes": 159147520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 127, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "78eb5f93f9975204fec8266c350d61fe6e0af00154590a18bff5adda82e32560", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 5376 - ] - }, - "dx": { - "sha256": "cfb4e34f9a5acfbd7f44094359cc37fb1653466897436cd847cf6c782f0fb62a", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 14336 - ] - }, - "dw": { - "sha256": "80417148de1919505236065672e995febe80f604c9acafe5c223aab71d624847", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 128, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5271827578544617e-05, - "max_error_over_tolerance": 0.0003049470251426101, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000971376895904541, - "max_error_over_tolerance": 0.009242621250450611, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.0865200087428093, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5223676562309265, - "samples_ms": [ - 0.572938472032547, - 0.5223676562309265, - 0.5280077457427979, - 0.5104504525661469, - 0.5200915038585663 - ], - "peak_allocated_bytes": 231346176, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.21956488490104675, - "samples_ms": [ - 0.23156777024269104, - 0.23876875638961792, - 0.21956488490104675, - 0.21862611174583435, - 0.2158135175704956 - ], - "peak_allocated_bytes": 234098688, - "incremental_peak_bytes": 4128768, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.19888579845428467, - "samples_ms": [ - 0.20590052008628845, - 0.19888579845428467, - 0.20147860050201416, - 0.19458308815956116, - 0.1957826316356659 - ], - "peak_allocated_bytes": 384110592, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.3782273530960083, - "samples_ms": [ - 5.160674452781677, - 3.3782273530960083, - 3.2818950712680817, - 3.037508577108383, - 4.338420927524567 - ], - "peak_allocated_bytes": 558444032, - "incremental_peak_bytes": 328474112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.10214373469352722, - "samples_ms": [ - 0.11356174945831299, - 0.10214373469352722, - 0.09890273213386536, - 0.11762231588363647, - 0.10176002979278564 - ], - "peak_allocated_bytes": 231346176, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8591338992118835, - "samples_ms": [ - 0.8280053734779358, - 0.8591338992118835, - 0.5269050598144531, - 0.8636452257633209, - 1.0673291981220245 - ], - "peak_allocated_bytes": 389156864, - "incremental_peak_bytes": 159186944, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 128, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5b77f873e5f379efd4a7ac6a396c7c73c851200f081d1b574928df23611350b3", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 5376 - ] - }, - "dx": { - "sha256": "3c37cb8907933fb2c047613b4ec6941d311a7fcf23484f52c90bedefcd9bf4fe", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 14336 - ] - }, - "dw": { - "sha256": "aba1da1cf968b1877fb75a15257693df005bd2fce5a66ce627eef4f30ae6ec9b", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5273690223693848e-05, - "max_error_over_tolerance": 0.0003049805236514658, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009714066982269287, - "max_error_over_tolerance": 0.009242905303835869, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.08660212904214859, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.4708394408226013, - "samples_ms": [ - 0.4997439682483673, - 0.47484785318374634, - 0.46965107321739197, - 0.4708394408226013, - 0.465981662273407 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2001151442527771, - "samples_ms": [ - 0.21700188517570496, - 0.204574316740036, - 0.19966810941696167, - 0.2001151442527771, - 0.19906088709831238 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 4161024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.23935362696647644, - "samples_ms": [ - 0.2619810402393341, - 0.23935362696647644, - 0.23012980818748474, - 0.2315528690814972, - 0.24047866463661194 - ], - "peak_allocated_bytes": 390986240, - "incremental_peak_bytes": 160946688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.217620939016342, - "samples_ms": [ - 4.787694662809372, - 4.113547503948212, - 4.1184090077877045, - 4.218190908432007, - 4.217620939016342 - ], - "peak_allocated_bytes": 564981248, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.10640174150466919, - "samples_ms": [ - 0.11917948722839355, - 0.10800361633300781, - 0.10640174150466919, - 0.10486319661140442, - 0.1055300235748291 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2598484754562378, - "samples_ms": [ - 1.3252384960651398, - 1.2041032314300537, - 1.2598484754562378, - 1.2480095028877258, - 1.3608895242214203 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 256, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 2.794712781906128e-05, - "max_error_over_tolerance": 0.0005570866051129997, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009754598140716553, - "max_error_over_tolerance": 0.009243669919669628, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.031119346618652344, - "max_error_over_tolerance": 0.11552537977695465, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5064867436885834, - "samples_ms": [ - 0.5556009709835052, - 0.5189701914787292, - 0.5064867436885834, - 0.4983171820640564, - 0.5018599331378937 - ], - "peak_allocated_bytes": 241442816, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.22333860397338867, - "samples_ms": [ - 0.23899972438812256, - 0.22518262267112732, - 0.22333860397338867, - 0.22138282656669617, - 0.21755322813987732 - ], - "peak_allocated_bytes": 246947840, - "incremental_peak_bytes": 8257536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2390630543231964, - "samples_ms": [ - 0.2594850957393646, - 0.24216622114181519, - 0.23720413446426392, - 0.2390630543231964, - 0.23523345589637756 - ], - "peak_allocated_bytes": 392830976, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.2709968984127045, - "samples_ms": [ - 3.2444484531879425, - 8.486852049827576, - 4.227753728628159, - 4.548773169517517, - 4.2709968984127045 - ], - "peak_allocated_bytes": 585521664, - "incremental_peak_bytes": 346831360, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1284368336200714, - "samples_ms": [ - 0.1284368336200714, - 0.13460591435432434, - 0.11734664440155029, - 0.12366101145744324, - 0.13120099902153015 - ], - "peak_allocated_bytes": 241442816, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2776143848896027, - "samples_ms": [ - 1.3336129486560822, - 1.2776143848896027, - 1.1073872447013855, - 1.1383816599845886, - 1.2937337160110474 - ], - "peak_allocated_bytes": 402923520, - "incremental_peak_bytes": 164233216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 4, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 4, - 224 - ] - }, - "canonical_active_rows": 256, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "d0be45f3fc497360fcdb5e7dff1866a56f1703c6e1a70564917739c537e09b1e", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 5376 - ] - }, - "dx": { - "sha256": "f1a528f3d88daf90bb860d5ef2f0f42176e65c0eff3950cf880db91352ea4961", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 14336 - ] - }, - "dw": { - "sha256": "bca753e4f35b9f53e4787e16c45a1a9f61b805cc52777b45d2aac47131e792cc", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 513, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 2.748146653175354e-05, - "max_error_over_tolerance": 0.0005478710518218577, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009688436985015869, - "max_error_over_tolerance": 0.009219937026500702, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.031253814697265625, - "max_error_over_tolerance": 0.11986027657985687, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.508233904838562, - "samples_ms": [ - 0.5344487726688385, - 0.5267933011054993, - 0.508233904838562, - 0.4960186779499054, - 0.48770755529403687 - ], - "peak_allocated_bytes": 261716480, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.36629289388656616, - "samples_ms": [ - 0.38085877895355225, - 0.36629289388656616, - 0.3698691725730896, - 0.36259740591049194, - 0.36022067070007324 - ], - "peak_allocated_bytes": 270909440, - "incremental_peak_bytes": 14708736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.3862045705318451, - "samples_ms": [ - 0.402599573135376, - 0.3907792270183563, - 0.3862045705318451, - 0.38257986307144165, - 0.3825165331363678 - ], - "peak_allocated_bytes": 432737792, - "incremental_peak_bytes": 176537088, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.5304372906684875, - "samples_ms": [ - 4.907950758934021, - 4.5304372906684875, - 4.573829472064972, - 4.422713071107864, - 4.3122731149196625 - ], - "peak_allocated_bytes": 661960704, - "incremental_peak_bytes": 405760000, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.19780173897743225, - "samples_ms": [ - 0.19539892673492432, - 0.24132058024406433, - 0.19425898790359497, - 0.19780173897743225, - 0.30922144651412964 - ], - "peak_allocated_bytes": 261716480, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.3085603713989258, - "samples_ms": [ - 1.078341156244278, - 1.030702143907547, - 1.5124604105949402, - 2.3444928228855133, - 1.3085603713989258 - ], - "peak_allocated_bytes": 430565888, - "incremental_peak_bytes": 174365184, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 9, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 9, - 224 - ] - }, - "canonical_active_rows": 513, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "e72ed22b203a375ad2b7bba287cdca983fa623986fd0409466097f088101d531", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 5376 - ] - }, - "dx": { - "sha256": "926625b6fdbe67eaaaa5ace548fe15d26850ee6ea2c3cfbfe503b42eba987e49", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 14336 - ] - }, - "dw": { - "sha256": "636678ed4d73fdd1ecab8a9b4eff1ad8eb879428c0fae9d0c8d6f9e557f886da", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 1024, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.0370429158210754e-05, - "max_error_over_tolerance": 0.0006054941914044321, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009716451168060303, - "max_error_over_tolerance": 0.009236982092261314, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06217002868652344, - "max_error_over_tolerance": 0.14585569500923157, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.954553484916687, - "samples_ms": [ - 1.0414011776447296, - 1.0842233896255493, - 0.954553484916687, - 0.93870609998703, - 0.9392276406288147 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.6548352539539337, - "samples_ms": [ - 0.6848238408565521, - 0.6539858877658844, - 0.6239674985408783, - 0.6548352539539337, - 0.6782747805118561 - ], - "peak_allocated_bytes": 320372736, - "incremental_peak_bytes": 29360128, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.6275102496147156, - "samples_ms": [ - 0.6384477019309998, - 0.6108023226261139, - 0.6275102496147156, - 0.6749443709850311, - 0.5772635340690613 - ], - "peak_allocated_bytes": 445153280, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.6306513249874115, - "samples_ms": [ - 4.335585981607437, - 4.6306513249874115, - 4.8942118883132935, - 4.492711275815964, - 4.805084317922592 - ], - "peak_allocated_bytes": 749822976, - "incremental_peak_bytes": 458810368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.2659261226654053, - "samples_ms": [ - 0.5080997943878174, - 0.2718046307563782, - 0.2659261226654053, - 0.26274845004081726, - 0.25920569896698 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2693442404270172, - "samples_ms": [ - 1.2693442404270172, - 1.1054575443267822, - 1.3971664011478424, - 1.0951757431030273, - 1.386839896440506 - ], - "peak_allocated_bytes": 485523456, - "incremental_peak_bytes": 194510848, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 16, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 16, - 224 - ] - }, - "canonical_active_rows": 1024, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "183d20b577e0b0e4377a7425c17b28baf3049214b9a0c5300a3c47d9b860af5a", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 5376 - ] - }, - "dx": { - "sha256": "33904fcbd1ec09dd48203f527771f9212638ae09c7d1a07f14fae5daad2ebec5", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 14336 - ] - }, - "dw": { - "sha256": "afd58776763f76651a948c597d8456937d93a28295c385bb00c94228382ed5f5", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 2048, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.036297857761383e-05, - "max_error_over_tolerance": 0.0006053456454537809, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009779930114746094, - "max_error_over_tolerance": 0.009302103891968727, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06254959106445312, - "max_error_over_tolerance": 0.14828670024871826, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.4133527874946594, - "samples_ms": [ - 1.5480369329452515, - 1.4133527874946594, - 1.4082454144954681, - 1.4134794473648071, - 1.3939887285232544 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 1.0775364935398102, - "samples_ms": [ - 1.0775364935398102, - 1.0779686272144318, - 1.081816852092743, - 1.0686367750167847, - 1.0738149285316467 - ], - "peak_allocated_bytes": 419495936, - "incremental_peak_bytes": 58720256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 1.0426081717014313, - "samples_ms": [ - 1.0580308735370636, - 1.044493168592453, - 1.0379701852798462, - 1.0426081717014313, - 1.042395830154419 - ], - "peak_allocated_bytes": 514916352, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.988609045743942, - "samples_ms": [ - 5.804240703582764, - 6.012089550495148, - 5.988609045743942, - 5.955345928668976, - 6.581451743841171 - ], - "peak_allocated_bytes": 970115072, - "incremental_peak_bytes": 609339392, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.4807375371456146, - "samples_ms": [ - 0.48383697867393494, - 0.4701539874076843, - 0.48117712140083313, - 0.4807375371456146, - 0.46753138303756714 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.5724487602710724, - "samples_ms": [ - 1.5830844640731812, - 1.5198811888694763, - 1.4572776854038239, - 1.6771145164966583, - 1.5724487602710724 - ], - "peak_allocated_bytes": 595656704, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 32, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 32, - 224 - ] - }, - "canonical_active_rows": 2048, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "e54e5c678f07f22f9b897444ecdc015f427004a906336492dbf908c6edd66611", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 5376 - ] - }, - "dx": { - "sha256": "79d3974647b5f69fc8d5765d88d92f95775a5d5ae99006c5c27bbba9143f0fc8", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 14336 - ] - }, - "dw": { - "sha256": "d3cace7e7fd59fe9eb28a725da4c1f3666f332ae5f7f167d9e60b61dd37df160", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 4096, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.0279159545898438e-05, - "max_error_over_tolerance": 0.0006036892300471663, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781420230865479, - "max_error_over_tolerance": 0.009296612814068794, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.11181640625, - "max_error_over_tolerance": 0.14843568205833435, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 2.7626194059848785, - "samples_ms": [ - 2.916518598794937, - 2.810914069414139, - 2.7626194059848785, - 2.624671906232834, - 2.6615671813488007 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 2.041943371295929, - "samples_ms": [ - 2.1330080926418304, - 2.076968550682068, - 2.041943371295929, - 1.9956156611442566, - 1.956440508365631 - ], - "peak_allocated_bytes": 617742336, - "incremental_peak_bytes": 117440512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 2.022635191679001, - "samples_ms": [ - 2.0455382764339447, - 2.022635191679001, - 2.0143650472164154, - 2.0569004118442535, - 2.008773386478424 - ], - "peak_allocated_bytes": 654442496, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 8.995618671178818, - "samples_ms": [ - 9.004231542348862, - 9.068693965673447, - 8.96860659122467, - 8.952312171459198, - 8.995618671178818 - ], - "peak_allocated_bytes": 1410699264, - "incremental_peak_bytes": 910397440, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.8829720318317413, - "samples_ms": [ - 0.947900116443634, - 0.895179808139801, - 0.8829720318317413, - 0.8808374404907227, - 0.8783228695392609 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 2.5516562163829803, - "samples_ms": [ - 2.5715231895446777, - 2.5516562163829803, - 2.6257187128067017, - 2.540022134780884, - 2.522353082895279 - ], - "peak_allocated_bytes": 815923200, - "incremental_peak_bytes": 315621376, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 64, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 64, - 224 - ] - }, - "canonical_active_rows": 4096, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "1621d18fce3171753ac375d7e7d597a615918a0a51baa4631a66f856526ae3a0", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 5376 - ] - }, - "dx": { - "sha256": "2aeba2c1c7b13af7512f076598a63022d6d4269c463c154594a6f2daadf1ef14", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 14336 - ] - }, - "dw": { - "sha256": "223f219ee9c8d92b08f096a064bed2e0e784a20f30010e3b4c96b74a17a51110", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 8192, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.0450522899627686e-05, - "max_error_over_tolerance": 0.0006070910603739321, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009778738021850586, - "max_error_over_tolerance": 0.009276843629777431, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12531661987304688, - "max_error_over_tolerance": 0.1686829924583435, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 6.8867504596710205, - "samples_ms": [ - 6.461631506681442, - 8.28661024570465, - 6.989728659391403, - 6.8867504596710205, - 6.5978094935417175 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 4.373438656330109, - "samples_ms": [ - 4.3046735227108, - 4.34340164065361, - 4.423592239618301, - 4.373438656330109, - 4.400052130222321 - ], - "peak_allocated_bytes": 1014235136, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 4.201643168926239, - "samples_ms": [ - 4.201643168926239, - 4.208642989397049, - 4.134170711040497, - 4.1950903832912445, - 4.259232431650162 - ], - "peak_allocated_bytes": 933494784, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 18.23057234287262, - "samples_ms": [ - 22.7372869849205, - 19.50514316558838, - 18.23057234287262, - 18.210425972938538, - 17.958156764507294 - ], - "peak_allocated_bytes": 2291867648, - "incremental_peak_bytes": 1512513536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 1.7067305743694305, - "samples_ms": [ - 1.721072942018509, - 1.7067305743694305, - 1.7101801931858063, - 1.706693321466446, - 1.7048045992851257 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 5.713455379009247, - "samples_ms": [ - 5.423523485660553, - 5.721472203731537, - 5.715426057577133, - 5.713455379009247, - 5.707930773496628 - ], - "peak_allocated_bytes": 1256456192, - "incremental_peak_bytes": 477102080, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 128, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 128, - 224 - ] - }, - "canonical_active_rows": 8192, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6a54a53cc4462816868b87f3f7a5d5ac1748a1de999f1904a2d3e9d7414644dc", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 5376 - ] - }, - "dx": { - "sha256": "caedce8f925825f99fe5013c75f3c840718bf79cf9c680cf55314447f6b088df", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 14336 - ] - }, - "dw": { - "sha256": "941d3b7ded8cbec8ede620b452d917b70c68b286c690e7640d206d66e24352cc", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 16385, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.045797348022461e-05, - "max_error_over_tolerance": 0.0006071952520869672, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009784698486328125, - "max_error_over_tolerance": 0.009305674582719803, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.2373809814453125, - "max_error_over_tolerance": 0.16933324933052063, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 20.00322937965393, - "samples_ms": [ - 20.033571869134903, - 20.11100947856903, - 19.97118815779686, - 20.00322937965393, - 19.972704350948334 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 8.884463459253311, - "samples_ms": [ - 8.922353386878967, - 8.921671658754349, - 8.82551446557045, - 8.884463459253311, - 8.845862001180649 - ], - "peak_allocated_bytes": 1807319040, - "incremental_peak_bytes": 469790720, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 9.360156953334808, - "samples_ms": [ - 9.191662073135376, - 9.159374982118607, - 9.420003741979599, - 9.573694318532944, - 9.360156953334808 - ], - "peak_allocated_bytes": 2138853376, - "incremental_peak_bytes": 801325056, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 43.33186149597168, - "samples_ms": [ - 46.373505145311356, - 42.94878989458084, - 43.254923075437546, - 44.764868915081024, - 43.33186149597168 - ], - "peak_allocated_bytes": 4701607936, - "incremental_peak_bytes": 3364079616, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 3.1894929707050323, - "samples_ms": [ - 3.1894929707050323, - 3.1828992068767548, - 3.18700447678566, - 3.405846655368805, - 3.1903572380542755 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 11.646967381238937, - "samples_ms": [ - 11.43326610326767, - 11.646967381238937, - 11.721748858690262, - 11.743105947971344, - 11.5024633705616 - ], - "peak_allocated_bytes": 2137631232, - "incremental_peak_bytes": 800102912, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 257, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 257, - 224 - ] - }, - "canonical_active_rows": 16385, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "aa4bf4200f549127eb85c3a288b7224d4a16b52a8043013d29d71f7be73284ef", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 5376 - ] - }, - "dx": { - "sha256": "8cd2308514c54aaadb5af89decef441854e45b7ba8b199e8acc5949f4c754650", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 14336 - ] - }, - "dw": { - "sha256": "430f8d5f6e7a36da69ea70998e41ade26ff44b9406cd9fe0f7d12222c4d74167", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 24577, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.0390918254852295e-05, - "max_error_over_tolerance": 0.0006057700957171619, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009780526161193848, - "max_error_over_tolerance": 0.009305845014750957, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.25164031982421875, - "max_error_over_tolerance": 0.18054932355880737, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 31.06134384870529, - "samples_ms": [ - 31.1649888753891, - 31.079281121492386, - 30.950598418712616, - 30.888359993696213, - 31.06134384870529 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 18.26496049761772, - "samples_ms": [ - 18.45216378569603, - 18.024198710918427, - 17.759855836629868, - 19.124291837215424, - 18.26496049761772 - ], - "peak_allocated_bytes": 2600304640, - "incremental_peak_bytes": 704671744, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 14.54518735408783, - "samples_ms": [ - 14.171525835990906, - 14.548677951097488, - 14.515917748212814, - 14.54518735408783, - 14.740563929080963 - ], - "peak_allocated_bytes": 3019919360, - "incremental_peak_bytes": 1124286464, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 71.44178822636604, - "samples_ms": [ - 71.44178822636604, - 70.59933245182037, - 71.16098329424858, - 73.56076687574387, - 73.6989676952362 - ], - "peak_allocated_bytes": 6786906112, - "incremental_peak_bytes": 4891273216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 4.727460443973541, - "samples_ms": [ - 4.791069775819778, - 4.766747355461121, - 4.727460443973541, - 4.718884825706482, - 4.720926284790039 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 18.98377016186714, - "samples_ms": [ - 19.223317503929138, - 18.994607031345367, - 18.97561177611351, - 18.98377016186714, - 18.50433275103569 - ], - "peak_allocated_bytes": 3018697216, - "incremental_peak_bytes": 1123064320, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 385, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 385, - 224 - ] - }, - "canonical_active_rows": 24577, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "4a4ae7db6de8e3bb9ea488f2da53afd2fcdd526febfc27e20f32db1696112375", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 5376 - ] - }, - "dx": { - "sha256": "fd4a3d555ad9b8e8a0be0a0ac212d337c20e4fd6e691ce1982b341ceb2818616", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 14336 - ] - }, - "dw": { - "sha256": "facb4ce8b0cb9c90f9451956631e2490da927d2a8ec36bf07eaf6b5c237f6cfe", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32768, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 3.0469149351119995e-05, - "max_error_over_tolerance": 0.0006074772099964321, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781122207641602, - "max_error_over_tolerance": 0.009311916306614876, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.252532958984375, - "max_error_over_tolerance": 0.18232759833335876, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 41.69951379299164, - "samples_ms": [ - 41.7838990688324, - 41.689883917570114, - 41.70183464884758, - 41.69951379299164, - 41.60373657941818 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 36.576222628355026, - "samples_ms": [ - 37.16915473341942, - 36.50326281785965, - 36.96649894118309, - 36.576222628355026, - 34.471482038497925 - ], - "peak_allocated_bytes": 3393191936, - "incremental_peak_bytes": 939524096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 18.744267523288727, - "samples_ms": [ - 18.744267523288727, - 20.046759396791458, - 18.29393580555916, - 18.44470575451851, - 19.663449376821518 - ], - "peak_allocated_bytes": 2607808512, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 106.80445283651352, - "samples_ms": [ - 108.48763585090637, - 104.52854260802269, - 109.95243117213249, - 106.80445283651352, - 105.52418604493141 - ], - "peak_allocated_bytes": 7578877952, - "incremental_peak_bytes": 5125210112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 6.743166595697403, - "samples_ms": [ - 6.233599036931992, - 6.239496171474457, - 6.743166595697403, - 7.567781955003738, - 7.592573761940002 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 20.594339817762375, - "samples_ms": [ - 22.5556418299675, - 21.25835418701172, - 20.080365240573883, - 20.07279545068741, - 20.594339817762375 - ], - "peak_allocated_bytes": 3899654144, - "incremental_peak_bytes": 1445986304, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 512, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 512, - 224 - ] - }, - "canonical_active_rows": 32768, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "fb0ebbf1abddf063f48ef33b3af3473a1c6e77dd25c71e53436c36bc27bf5633", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 5376 - ] - }, - "dx": { - "sha256": "5500147da973aec9b668136996f3cf618885a6a54a3293528cd0c5b8382c6f59", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 14336 - ] - }, - "dw": { - "sha256": "4fa5b8c06544657f604f5ccfeca21afbae4284d9ed941aef981c0098a0c7b7ca", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - } - ] - } - } - } -} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-random.json b/docs/operators/evidence/h3-down-h100-20261010/h100-random.json deleted file mode 100644 index 11e59cb1b..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/h100-random.json +++ /dev/null @@ -1,10459 +0,0 @@ -{ - "schema_version": 1, - "gpu": "h100", - "provenance": { - "derivative": "Sanitized historical evidence; only execution paths replaced with portable placeholders. All measurements and findings retained. No new GPU run.", - "original_commit": "006021ca9875cdc4ae66af7ed184514070f3274a", - "original_path": "docs/operators/evidence/h3-down-h100-20261010/h100-random.json" - }, - "artifacts": { - "h100-random.json": { - "original_sha256": "97652ca2c83ba42e821a98fbc2810000152f22af2dffb826b433ce6eabcad1b0", - "format": "json", - "content": { - "schema": "h3-down-measured-case-part-v1", - "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", - "case_indexes": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15 - ], - "cases": [ - { - "rows": 1, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0077440738677978516, - "max_error_over_tolerance": 0.08342820405960083, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000848233699798584, - "max_error_over_tolerance": 0.008046787232160568, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.0324886292219162, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.4641115665435791, - "samples_ms": [ - 0.5014799535274506, - 0.4641115665435791, - 0.4806891083717346, - 0.46247243881225586, - 0.46306103467941284 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19700080156326294, - "samples_ms": [ - 0.19984692335128784, - 0.21108612418174744, - 0.19700080156326294, - 0.18755346536636353, - 0.18952041864395142 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.16470998525619507, - "samples_ms": [ - 0.18192827701568604, - 0.1676119863986969, - 0.16402080655097961, - 0.16470998525619507, - 0.1600421965122223 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.881074815988541, - "samples_ms": [ - 10.020162910223007, - 4.396550357341766, - 4.599045962095261, - 5.881074815988541, - 7.008261978626251 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08215755224227905, - "samples_ms": [ - 0.107593834400177, - 0.08467957377433777, - 0.08215755224227905, - 0.07981061935424805, - 0.07973238825798035 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2091360986232758, - "samples_ms": [ - 0.8774884045124054, - 1.2091360986232758, - 1.040332019329071, - 1.277294009923935, - 1.5825442969799042 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 31, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014126777648925781, - "max_error_over_tolerance": 0.10740139335393906, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009508132934570312, - "max_error_over_tolerance": 0.009040186181664467, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007812023162841797, - "max_error_over_tolerance": 0.055731140077114105, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6497353315353394, - "samples_ms": [ - 0.6950311362743378, - 0.6497353315353394, - 0.6531104445457458, - 0.6261691451072693, - 0.6205104291439056 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.28938427567481995, - "samples_ms": [ - 0.31585991382598877, - 0.29233843088150024, - 0.2748854458332062, - 0.2711452543735504, - 0.28938427567481995 - ], - "peak_allocated_bytes": 224250880, - "incremental_peak_bytes": 888832, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.3261081874370575, - "samples_ms": [ - 0.35964325070381165, - 0.3442540764808655, - 0.31934306025505066, - 0.3134310245513916, - 0.3261081874370575 - ], - "peak_allocated_bytes": 378764288, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.858814179897308, - "samples_ms": [ - 4.203196614980698, - 4.415713250637054, - 4.858814179897308, - 6.295427680015564, - 5.825243890285492 - ], - "peak_allocated_bytes": 537462784, - "incremental_peak_bytes": 314100736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1394413411617279, - "samples_ms": [ - 0.1394413411617279, - 3.767944872379303, - 0.316668301820755, - 0.11179596185684204, - 0.10490044951438904 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.3986565172672272, - "samples_ms": [ - 1.3986565172672272, - 1.439116895198822, - 1.512467861175537, - 1.0717213153839111, - 0.9977258741855621 - ], - "peak_allocated_bytes": 378724864, - "incremental_peak_bytes": 155362816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 31, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "bc382a772a452fbea1b5ee787fb4e0b65064e260c534136fbb25491a328f9d86", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 5376 - ] - }, - "dx": { - "sha256": "6266236128cb9315856b07e8bb3b991103c0067b4ceead50206f91bb9eab6975", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 14336 - ] - }, - "dw": { - "sha256": "fab683c8375ddd2d5031ab580471c42c5cb0194aeda91773d51337366aa21321", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014126777648925781, - "max_error_over_tolerance": 0.10740139335393906, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009508132934570312, - "max_error_over_tolerance": 0.009040186181664467, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007812738418579102, - "max_error_over_tolerance": 0.05573455989360809, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.48016756772994995, - "samples_ms": [ - 0.5154013633728027, - 0.49382075667381287, - 0.48016756772994995, - 0.4757493734359741, - 0.4628002643585205 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19780173897743225, - "samples_ms": [ - 0.21325424313545227, - 0.20276755094528198, - 0.19780173897743225, - 0.19660592079162598, - 0.19564852118492126 - ], - "peak_allocated_bytes": 224347648, - "incremental_peak_bytes": 917504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.14327839016914368, - "samples_ms": [ - 0.15502050518989563, - 0.14247745275497437, - 0.14327839016914368, - 0.1337788999080658, - 0.14632940292358398 - ], - "peak_allocated_bytes": 377570816, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.488680511713028, - "samples_ms": [ - 4.308421164751053, - 4.495076835155487, - 5.520697683095932, - 4.234891384840012, - 4.488680511713028 - ], - "peak_allocated_bytes": 536416256, - "incremental_peak_bytes": 312986112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09065121412277222, - "samples_ms": [ - 0.10181963443756104, - 0.09404495358467102, - 0.09065121412277222, - 0.08993595838546753, - 0.08934363722801208 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.9762495756149292, - "samples_ms": [ - 1.1316165328025818, - 0.9762495756149292, - 1.1035725474357605, - 0.894542783498764, - 0.8411481976509094 - ], - "peak_allocated_bytes": 378832384, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 32, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "b5d3d5e2880a7ce138593c82d8e6f7903f4b6eee7e6fea9c4347ff0e13819f71", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 5376 - ] - }, - "dx": { - "sha256": "d7424a7f439bb3784a9fd0c29fdfed30911cc028598d1763906f4ede9c700fdc", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 14336 - ] - }, - "dw": { - "sha256": "5afe57d24a47021447aea4bdb28d19646b771e50129c64b65b697673ba4e66d7", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014126300811767578, - "max_error_over_tolerance": 0.10739777982234955, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009508132934570312, - "max_error_over_tolerance": 0.009040186181664467, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.055741358548402786, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5179569125175476, - "samples_ms": [ - 0.5485452711582184, - 0.5214326083660126, - 0.5179569125175476, - 0.5115382373332977, - 0.5022026598453522 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2129785716533661, - "samples_ms": [ - 0.2353973686695099, - 0.22062286734580994, - 0.2129785716533661, - 0.2116560935974121, - 0.20866096019744873 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.22759661078453064, - "samples_ms": [ - 0.24954602122306824, - 0.22759661078453064, - 0.22070109844207764, - 0.23080036044120789, - 0.22421404719352722 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.993552178144455, - "samples_ms": [ - 3.993552178144455, - 3.9886869490146637, - 4.171747714281082, - 3.9090216159820557, - 4.432704299688339 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08925795555114746, - "samples_ms": [ - 0.09898096323013306, - 0.09119883179664612, - 0.08878856897354126, - 0.08925795555114746, - 0.08856505155563354 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.4379099011421204, - "samples_ms": [ - 5.710933357477188, - 2.139914780855179, - 1.4379099011421204, - 0.9647868573665619, - 1.1961422860622406 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 127, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015645503997802734, - "max_error_over_tolerance": 0.12006104737520218, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.0866260752081871, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5102045834064484, - "samples_ms": [ - 0.5356669425964355, - 0.5090311169624329, - 0.4749707877635956, - 0.5102045834064484, - 0.5424767732620239 - ], - "peak_allocated_bytes": 231267328, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.22759288549423218, - "samples_ms": [ - 0.2511031925678253, - 0.2266913652420044, - 0.2282075583934784, - 0.22707507014274597, - 0.22759288549423218 - ], - "peak_allocated_bytes": 233998336, - "incremental_peak_bytes": 4096512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.21474435925483704, - "samples_ms": [ - 0.23312494158744812, - 0.21474435925483704, - 0.21308287978172302, - 0.2142675220966339, - 0.2564825117588043 - ], - "peak_allocated_bytes": 389515264, - "incremental_peak_bytes": 159613440, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.306524991989136, - "samples_ms": [ - 4.206404089927673, - 3.9475373923778534, - 4.449903964996338, - 11.210303753614426, - 4.306524991989136 - ], - "peak_allocated_bytes": 563264512, - "incremental_peak_bytes": 333362688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09214133024215698, - "samples_ms": [ - 0.11442974209785461, - 0.09933486580848694, - 0.09214133024215698, - 0.09178370237350464, - 0.09139999747276306 - ], - "peak_allocated_bytes": 231267328, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.6517879664897919, - "samples_ms": [ - 0.8602924644947052, - 0.738535076379776, - 0.6517879664897919, - 0.4775114357471466, - 0.48815831542015076 - ], - "peak_allocated_bytes": 389049344, - "incremental_peak_bytes": 159147520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 127, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "7ba29d94e89afc0b879e4f3a22d64b0d62442a747d28d4c913f7fe6456fbf347", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 5376 - ] - }, - "dx": { - "sha256": "296804066060eabb8d1b14ed7dbc3e4ecbb2315955292acf1c08f818fcef32c5", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 14336 - ] - }, - "dw": { - "sha256": "de7318bed0fb5661d902180de7980b37e9a6cc78f8a6bfa5e098613fe80a7dcb", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 128, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015645503997802734, - "max_error_over_tolerance": 0.12006104737520218, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.0865200087428093, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5424097180366516, - "samples_ms": [ - 0.5952082574367523, - 0.5424097180366516, - 0.5490556359291077, - 0.5400031805038452, - 0.5214028060436249 - ], - "peak_allocated_bytes": 231346176, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.22576004266738892, - "samples_ms": [ - 0.25213882327079773, - 0.2295970916748047, - 0.22279098629951477, - 0.22278353571891785, - 0.22576004266738892 - ], - "peak_allocated_bytes": 234098688, - "incremental_peak_bytes": 4128768, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.210687518119812, - "samples_ms": [ - 0.22293627262115479, - 0.210687518119812, - 0.20804256200790405, - 0.20571425557136536, - 0.21763518452644348 - ], - "peak_allocated_bytes": 384110592, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.423797130584717, - "samples_ms": [ - 3.662385046482086, - 4.423797130584717, - 4.459649324417114, - 4.2327046394348145, - 4.465457051992416 - ], - "peak_allocated_bytes": 558444032, - "incremental_peak_bytes": 328474112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09127333760261536, - "samples_ms": [ - 0.121239572763443, - 0.09503215551376343, - 0.09127333760261536, - 0.09118765592575073, - 0.0912398099899292 - ], - "peak_allocated_bytes": 231346176, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.309465616941452, - "samples_ms": [ - 1.0634250938892365, - 1.309465616941452, - 1.2851320207118988, - 1.330547034740448, - 1.3811662793159485 - ], - "peak_allocated_bytes": 389156864, - "incremental_peak_bytes": 159186944, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 128, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6407e4fc7b31d0fb6bc7983e9b73af0ed6865d38a996701f28dd5b96653b4316", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 5376 - ] - }, - "dx": { - "sha256": "0a8d2486c11b7036a22b52461a48637df6db771a88ca37c15917e3497c33195e", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 14336 - ] - }, - "dw": { - "sha256": "97560750faf93915f48fff41bf45f3f36d5544ee992bc04f79a7a54613bd477c", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01564645767211914, - "max_error_over_tolerance": 0.12006834149360657, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625476837158203, - "max_error_over_tolerance": 0.08660759031772614, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5118474364280701, - "samples_ms": [ - 0.5456544458866119, - 0.5125962197780609, - 0.5012191832065582, - 0.5118474364280701, - 0.5014054477214813 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.22323429584503174, - "samples_ms": [ - 0.2366490662097931, - 0.22323429584503174, - 0.21626055240631104, - 0.22421032190322876, - 0.21079927682876587 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 4161024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2612210810184479, - "samples_ms": [ - 0.29610469937324524, - 0.2780333161354065, - 0.25999173521995544, - 0.2612210810184479, - 0.24727359414100647 - ], - "peak_allocated_bytes": 390986240, - "incremental_peak_bytes": 160946688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.367075860500336, - "samples_ms": [ - 9.567219763994217, - 4.193499684333801, - 4.239816218614578, - 4.367075860500336, - 4.3790265917778015 - ], - "peak_allocated_bytes": 564981248, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11000409722328186, - "samples_ms": [ - 0.122804194688797, - 0.11217594146728516, - 0.10993704199790955, - 0.10891258716583252, - 0.11000409722328186 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.257181167602539, - "samples_ms": [ - 1.0933689773082733, - 1.257181167602539, - 1.3633482158184052, - 1.405365765094757, - 0.9644664824008942 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 256, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015221118927001953, - "max_error_over_tolerance": 0.11568786203861237, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009686648845672607, - "max_error_over_tolerance": 0.009199724532663822, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.031119346618652344, - "max_error_over_tolerance": 0.11714421957731247, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.4711635410785675, - "samples_ms": [ - 0.49818307161331177, - 0.4711635410785675, - 0.4619285464286804, - 0.4760324954986572, - 0.46869367361068726 - ], - "peak_allocated_bytes": 241442816, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.23234263062477112, - "samples_ms": [ - 0.2589598298072815, - 0.23838505148887634, - 0.2281591296195984, - 0.23234263062477112, - 0.22763758897781372 - ], - "peak_allocated_bytes": 246947840, - "incremental_peak_bytes": 8257536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2569854259490967, - "samples_ms": [ - 0.27580931782722473, - 0.2569854259490967, - 0.26258453726768494, - 0.2506449818611145, - 0.24703145027160645 - ], - "peak_allocated_bytes": 392830976, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.024874418973923, - "samples_ms": [ - 3.8254931569099426, - 4.024874418973923, - 4.399307072162628, - 4.058755934238434, - 3.2460279762744904 - ], - "peak_allocated_bytes": 585521664, - "incremental_peak_bytes": 346831360, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1119375228881836, - "samples_ms": [ - 0.12212246656417847, - 0.11330097913742065, - 0.11178478598594666, - 0.1119375228881836, - 0.11095404624938965 - ], - "peak_allocated_bytes": 241442816, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.44563785195350647, - "samples_ms": [ - 0.8560195565223694, - 0.44563785195350647, - 0.43587759137153625, - 1.0146237909793854, - 0.43319910764694214 - ], - "peak_allocated_bytes": 402923520, - "incremental_peak_bytes": 164233216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 4, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 4, - 224 - ] - }, - "canonical_active_rows": 256, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "3659607138418e29f2f698deae3d17b367d227d3c6ea67dffa049e59a3feb8f4", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 5376 - ] - }, - "dx": { - "sha256": "8e6c64afeb2a469c4de837c5e087a30bf3c3cc619e78e45726feadd328c33b5f", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 14336 - ] - }, - "dw": { - "sha256": "f407bc90aa87075f4470ac7fac09940e27cbd0004a6c3b1f8967a811dd95d358", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 513, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015606880187988281, - "max_error_over_tolerance": 0.11976537108421326, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009740591049194336, - "max_error_over_tolerance": 0.009257806465029716, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.03125476837158203, - "max_error_over_tolerance": 0.11986027657985687, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5597323179244995, - "samples_ms": [ - 0.6031058728694916, - 0.5706027150154114, - 0.5584210157394409, - 0.5597323179244995, - 0.5541481077671051 - ], - "peak_allocated_bytes": 261716480, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.3941841423511505, - "samples_ms": [ - 0.4095993936061859, - 0.401180237531662, - 0.3921687602996826, - 0.3941841423511505, - 0.3767870366573334 - ], - "peak_allocated_bytes": 270909440, - "incremental_peak_bytes": 14708736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.4532076418399811, - "samples_ms": [ - 0.4839673638343811, - 0.4532076418399811, - 0.4369504749774933, - 0.46051666140556335, - 0.43874233961105347 - ], - "peak_allocated_bytes": 432737792, - "incremental_peak_bytes": 176537088, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.67793270945549, - "samples_ms": [ - 4.385463893413544, - 4.415247589349747, - 4.6799443662166595, - 4.7153085470199585, - 4.67793270945549 - ], - "peak_allocated_bytes": 661960704, - "incremental_peak_bytes": 405760000, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.20292028784751892, - "samples_ms": [ - 0.20292028784751892, - 0.19189715385437012, - 0.20329654216766357, - 0.2216733992099762, - 0.18816441297531128 - ], - "peak_allocated_bytes": 261716480, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.4380626380443573, - "samples_ms": [ - 1.453697681427002, - 1.389387995004654, - 1.4380626380443573, - 1.496635377407074, - 1.407429575920105 - ], - "peak_allocated_bytes": 430565888, - "incremental_peak_bytes": 174365184, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 9, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 9, - 224 - ] - }, - "canonical_active_rows": 513, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "3a6f35aed6e271d33e0a7cc5acd38e4ab30af9c34f441f93ca727ca7f90b9338", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 5376 - ] - }, - "dx": { - "sha256": "6695834948f5dc214db7e6893ad879700193dc7e6bc82bcb42a9954b7f60a609", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 14336 - ] - }, - "dw": { - "sha256": "cc181cb9f40cbe14e4b49f08e7a81ca61fbbe6f97fd291241596085501f961cd", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 1024, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015606880187988281, - "max_error_over_tolerance": 0.11976537108421326, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009763240814208984, - "max_error_over_tolerance": 0.009276159107685089, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06217002868652344, - "max_error_over_tolerance": 0.14585569500923157, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.0156221687793732, - "samples_ms": [ - 1.071583479642868, - 1.0156221687793732, - 1.0580532252788544, - 0.9893365204334259, - 0.9548701345920563 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.6798282265663147, - "samples_ms": [ - 0.686943531036377, - 0.6731189787387848, - 0.6798282265663147, - 0.697501003742218, - 0.6656236946582794 - ], - "peak_allocated_bytes": 320372736, - "incremental_peak_bytes": 29360128, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.5863979458808899, - "samples_ms": [ - 0.6058663129806519, - 0.5968138575553894, - 0.5863979458808899, - 0.580664724111557, - 0.5841217935085297 - ], - "peak_allocated_bytes": 445153280, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.7401003539562225, - "samples_ms": [ - 4.143659025430679, - 4.2868852615356445, - 4.7401003539562225, - 4.927046597003937, - 12.686174362897873 - ], - "peak_allocated_bytes": 749822976, - "incremental_peak_bytes": 458810368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.2707056701183319, - "samples_ms": [ - 0.2880282700061798, - 0.2771615982055664, - 0.2707056701183319, - 0.26961416006088257, - 0.26987865567207336 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.7799342274665833, - "samples_ms": [ - 0.8296966552734375, - 0.7799342274665833, - 0.7755793631076813, - 0.7859431207180023, - 0.7728338241577148 - ], - "peak_allocated_bytes": 485523456, - "incremental_peak_bytes": 194510848, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 16, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 16, - 224 - ] - }, - "canonical_active_rows": 1024, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "63b5882d73e482b20dcdba60ad939e2adf576ad4a6deed18640700a874292db2", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 5376 - ] - }, - "dx": { - "sha256": "75f218d54abf82a3e40b1e6d7c4a9b7301826c7680cfa714679a0dd9df69b7a5", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 14336 - ] - }, - "dw": { - "sha256": "0f7c2b049a78f21d8103882afc0b7de005c54445911cc0de881766436fbf48e7", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 2048, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01564931869506836, - "max_error_over_tolerance": 0.11999168992042542, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009767413139343262, - "max_error_over_tolerance": 0.00929025188088417, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06254959106445312, - "max_error_over_tolerance": 0.14829984307289124, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.5788264572620392, - "samples_ms": [ - 1.6528479754924774, - 1.5788264572620392, - 1.5440359711647034, - 1.575414091348648, - 2.727128565311432 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 1.134667545557022, - "samples_ms": [ - 1.8423348665237427, - 1.1614300310611725, - 1.1234171688556671, - 1.134667545557022, - 1.126706600189209 - ], - "peak_allocated_bytes": 419495936, - "incremental_peak_bytes": 58720256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 1.0633394122123718, - "samples_ms": [ - 1.082349568605423, - 1.064099371433258, - 1.0633394122123718, - 1.0608956217765808, - 1.055743545293808 - ], - "peak_allocated_bytes": 514916352, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 6.045274436473846, - "samples_ms": [ - 5.990024656057358, - 6.045274436473846, - 6.000753492116928, - 6.2157511711120605, - 6.278932094573975 - ], - "peak_allocated_bytes": 970115072, - "incremental_peak_bytes": 609339392, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.5051419138908386, - "samples_ms": [ - 0.5087107419967651, - 0.5051419138908386, - 0.48294663429260254, - 0.5096010863780975, - 0.4931837320327759 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.4390572905540466, - "samples_ms": [ - 1.4390572905540466, - 1.6509965062141418, - 1.4427416026592255, - 1.3899244368076324, - 1.398913562297821 - ], - "peak_allocated_bytes": 595656704, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 32, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 32, - 224 - ] - }, - "canonical_active_rows": 2048, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "248d6c0ce328417c18509b2e02a46d8227a6d94fc870be78561c534b7151c9f5", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 5376 - ] - }, - "dx": { - "sha256": "cbfbca4476a162e57b24cffd7aab8eed3e7b295a7d34bb6b2364fabf12132b07", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 14336 - ] - }, - "dw": { - "sha256": "660d931eee7dfe0d3da99d6052a9bd97a484ec62633b5fc358ecba64f30c4103", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 4096, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015695571899414062, - "max_error_over_tolerance": 0.1199989914894104, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781420230865479, - "max_error_over_tolerance": 0.009296612814068794, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.1242218017578125, - "max_error_over_tolerance": 0.167305588722229, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 2.7483180165290833, - "samples_ms": [ - 2.8183721005916595, - 2.7756132185459137, - 2.7483180165290833, - 2.717547118663788, - 2.71620973944664 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 2.094157040119171, - "samples_ms": [ - 2.1256916224956512, - 2.094157040119171, - 2.104472368955612, - 2.067010849714279, - 2.069532871246338 - ], - "peak_allocated_bytes": 617742336, - "incremental_peak_bytes": 117440512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 2.0535029470920563, - "samples_ms": [ - 2.072621136903763, - 2.0535029470920563, - 1.989707350730896, - 2.037379890680313, - 2.0740963518619537 - ], - "peak_allocated_bytes": 654442496, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 9.550560265779495, - "samples_ms": [ - 9.37473401427269, - 9.506858885288239, - 9.550560265779495, - 9.798500686883926, - 9.749151766300201 - ], - "peak_allocated_bytes": 1410699264, - "incremental_peak_bytes": 910397440, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.8845888078212738, - "samples_ms": [ - 0.9028799831867218, - 0.8845888078212738, - 0.8811093866825104, - 0.8833743631839752, - 0.8864738047122955 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 2.771172672510147, - "samples_ms": [ - 2.709783613681793, - 2.771172672510147, - 2.711579203605652, - 8.380550891160965, - 5.2012912929058075 - ], - "peak_allocated_bytes": 815923200, - "incremental_peak_bytes": 315621376, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 64, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 64, - 224 - ] - }, - "canonical_active_rows": 4096, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "8953d5c75afec1f337c637dd11f465879f3927856f98fbd5510996f20a07e4c7", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 5376 - ] - }, - "dx": { - "sha256": "fd023c1794371e2b1d28e1f1f534b585d47d3200b701103a0cdfecf11a5687ad", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 14336 - ] - }, - "dw": { - "sha256": "dfd59a6bfc69e8afe7672c2e883f2374938802dd8ea6e4be49c3df10b6e6e6bc", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 8192, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015656471252441406, - "max_error_over_tolerance": 0.11954245716333389, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000977635383605957, - "max_error_over_tolerance": 0.009301135316491127, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12531661987304688, - "max_error_over_tolerance": 0.1686829924583435, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 7.1127749979496, - "samples_ms": [ - 7.472507655620575, - 7.3508769273757935, - 7.102977484464645, - 7.1127749979496, - 6.655432283878326 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 4.442032426595688, - "samples_ms": [ - 4.521660506725311, - 4.305578768253326, - 4.4516921043396, - 4.442032426595688, - 4.309028387069702 - ], - "peak_allocated_bytes": 1014235136, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 4.242591559886932, - "samples_ms": [ - 4.133425652980804, - 4.224028438329697, - 4.242591559886932, - 4.332114011049271, - 4.297386854887009 - ], - "peak_allocated_bytes": 933494784, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 18.447231501340866, - "samples_ms": [ - 18.785148859024048, - 18.268324434757233, - 18.301360309123993, - 19.21449601650238, - 18.447231501340866 - ], - "peak_allocated_bytes": 2291867648, - "incremental_peak_bytes": 1512513536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 1.7193928360939026, - "samples_ms": [ - 1.7297640442848206, - 1.718033105134964, - 1.7193928360939026, - 1.7182528972625732, - 1.7548836767673492 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 5.222029983997345, - "samples_ms": [ - 5.2265264093875885, - 5.222029983997345, - 5.225643515586853, - 5.210563540458679, - 5.208142101764679 - ], - "peak_allocated_bytes": 1256456192, - "incremental_peak_bytes": 477102080, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 128, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 128, - 224 - ] - }, - "canonical_active_rows": 8192, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 5376 - ] - }, - "dx": { - "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 14336 - ] - }, - "dw": { - "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 16385, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015697956085205078, - "max_error_over_tolerance": 0.11958968639373779, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000978320837020874, - "max_error_over_tolerance": 0.00930879544466734, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.2373809814453125, - "max_error_over_tolerance": 0.16933324933052063, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 19.976794719696045, - "samples_ms": [ - 20.11549100279808, - 20.03374695777893, - 19.976794719696045, - 19.92051675915718, - 19.936978816986084 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 8.871227502822876, - "samples_ms": [ - 8.871227502822876, - 8.881252259016037, - 8.826285600662231, - 8.820831775665283, - 8.926533162593842 - ], - "peak_allocated_bytes": 1807319040, - "incremental_peak_bytes": 469790720, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 9.569797664880753, - "samples_ms": [ - 9.343385696411133, - 9.544700384140015, - 9.606815874576569, - 9.950287640094757, - 9.569797664880753 - ], - "peak_allocated_bytes": 2138853376, - "incremental_peak_bytes": 801325056, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 43.10831427574158, - "samples_ms": [ - 43.07585582137108, - 43.23599115014076, - 45.22192105650902, - 43.10831427574158, - 43.03518310189247 - ], - "peak_allocated_bytes": 4701607936, - "incremental_peak_bytes": 3364079616, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 3.1672269105911255, - "samples_ms": [ - 3.1685233116149902, - 3.171522170305252, - 3.1575746834278107, - 3.1672269105911255, - 3.161471337080002 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 11.300873011350632, - "samples_ms": [ - 11.613503098487854, - 12.109711766242981, - 11.300873011350632, - 11.228453367948532, - 11.10968366265297 - ], - "peak_allocated_bytes": 2137631232, - "incremental_peak_bytes": 800102912, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 257, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 257, - 224 - ] - }, - "canonical_active_rows": 16385, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "c4a1b1c7e38ae17487e7ef2238f3c3f2bd323df574ffb9996432a3598df29ace", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 5376 - ] - }, - "dx": { - "sha256": "bc96cb96333511e4ecf5dbc95fbe5beab1bf830f2d9cf4cee004c2fce6340052", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 14336 - ] - }, - "dw": { - "sha256": "a30ac927fbd5c9d1222c651d76e0fd188e336b7240d74d71f27407e2013c1cbf", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 24577, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015681743621826172, - "max_error_over_tolerance": 0.12001358717679977, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009780824184417725, - "max_error_over_tolerance": 0.009305674582719803, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.25164031982421875, - "max_error_over_tolerance": 0.18054932355880737, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 30.980128794908524, - "samples_ms": [ - 30.980128794908524, - 31.55752643942833, - 32.05243498086929, - 30.90331330895424, - 30.965466052293777 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 18.71088147163391, - "samples_ms": [ - 20.349707454442978, - 19.794635474681854, - 18.71088147163391, - 16.914289444684982, - 16.702190041542053 - ], - "peak_allocated_bytes": 2600304640, - "incremental_peak_bytes": 704671744, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 14.257971197366714, - "samples_ms": [ - 14.178819954395294, - 13.901382684707642, - 14.257971197366714, - 16.387518495321274, - 16.816169023513794 - ], - "peak_allocated_bytes": 3019919360, - "incremental_peak_bytes": 1124286464, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 72.33802601695061, - "samples_ms": [ - 70.44867426156998, - 72.74215668439865, - 73.1935054063797, - 72.33802601695061, - 69.4698803126812 - ], - "peak_allocated_bytes": 6786906112, - "incremental_peak_bytes": 4891273216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 4.81795147061348, - "samples_ms": [ - 4.900552332401276, - 4.775349050760269, - 4.790332168340683, - 4.81795147061348, - 4.820369184017181 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 18.448282033205032, - "samples_ms": [ - 16.797229647636414, - 17.87669211626053, - 18.863487988710403, - 18.856793642044067, - 18.448282033205032 - ], - "peak_allocated_bytes": 3018697216, - "incremental_peak_bytes": 1123064320, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 385, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 385, - 224 - ] - }, - "canonical_active_rows": 24577, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "36be331ffee01449374ae65b90be093f4f859b8d8e2e4a181e6e280375a4e0b9", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 5376 - ] - }, - "dx": { - "sha256": "f407c423a0726b739f5470ddd7ea438cb2532e9482f4a69f6f64853fdc2bf62c", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 14336 - ] - }, - "dw": { - "sha256": "1b384c414dc742aee13cd82e76a9a3824d8f8287a7821a64116e98240500cbfd", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32768, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01569652557373047, - "max_error_over_tolerance": 0.12013405561447144, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781122207641602, - "max_error_over_tolerance": 0.009311916306614876, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.2525482177734375, - "max_error_over_tolerance": 0.18232759833335876, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 41.62594676017761, - "samples_ms": [ - 41.61025583744049, - 41.62594676017761, - 41.63192957639694, - 41.6228361427784, - 41.692305356264114 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 35.47533228993416, - "samples_ms": [ - 33.461909741163254, - 34.31571647524834, - 37.22900524735451, - 35.47533228993416, - 35.92732176184654 - ], - "peak_allocated_bytes": 3393191936, - "incremental_peak_bytes": 939524096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 19.52551305294037, - "samples_ms": [ - 19.52551305294037, - 19.28829774260521, - 17.499323934316635, - 20.080339163541794, - 20.596228539943695 - ], - "peak_allocated_bytes": 2607808512, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 107.59353637695312, - "samples_ms": [ - 109.32975634932518, - 107.23195970058441, - 106.83485865592957, - 107.59353637695312, - 112.38069087266922 - ], - "peak_allocated_bytes": 7578877952, - "incremental_peak_bytes": 5125210112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 7.167089730501175, - "samples_ms": [ - 6.37710839509964, - 6.479129195213318, - 7.427748292684555, - 7.846798747777939, - 7.167089730501175 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 21.537311375141144, - "samples_ms": [ - 22.03044295310974, - 21.537311375141144, - 20.823486149311066, - 21.31655439734459, - 21.96405827999115 - ], - "peak_allocated_bytes": 3899654144, - "incremental_peak_bytes": 1445986304, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 512, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 512, - 224 - ] - }, - "canonical_active_rows": 32768, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 5376 - ] - }, - "dx": { - "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 14336 - ] - }, - "dw": { - "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - } - ] - } - } - } -} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json b/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json deleted file mode 100644 index d9e84283b..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json +++ /dev/null @@ -1,11164 +0,0 @@ -{ - "schema_version": 1, - "gpu": "h100", - "provenance": { - "derivative": "Sanitized historical evidence; only execution paths replaced with portable placeholders. All measurements and findings retained. No new GPU run.", - "original_commit": "006021ca9875cdc4ae66af7ed184514070f3274a", - "original_path": "docs/operators/evidence/h3-down-h100-20261010/h100-replays.json" - }, - "artifacts": { - "h100-replays.json": { - "original_sha256": "e47d4173b52ff3b2d68b314227710e91896c9753b2c485942bb4c585cf2cc89f", - "format": "json", - "content": { - "schema": "h3-down-measured-replays-v1", - "cold_cache": { - "schema": "h3-down-qualification-v2", - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "status": "passed", - "cases": [ - { - "rows": 1, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0077440738677978516, - "max_error_over_tolerance": 0.08342820405960083, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000848233699798584, - "max_error_over_tolerance": 0.008046787232160568, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.0324886292219162, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6608329713344574, - "samples_ms": [ - 0.7434599101543427, - 0.6608329713344574, - 0.6479956209659576 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.28777867555618286, - "samples_ms": [ - 0.31236186623573303, - 0.28777867555618286, - 0.2872943878173828 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.35877153277397156, - "samples_ms": [ - 0.35877153277397156, - 0.3423057496547699, - 0.40040165185928345 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 2.3678094148635864, - "samples_ms": [ - 2.812262624502182, - 2.3678094148635864, - 2.308174967765808 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08248910307884216, - "samples_ms": [ - 0.10496005415916443, - 0.08248910307884216, - 0.08038058876991272 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.0824166238307953, - "samples_ms": [ - 1.0685622692108154, - 1.1040419340133667, - 1.0824166238307953 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014126300811767578, - "max_error_over_tolerance": 0.10739777982234955, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009508132934570312, - "max_error_over_tolerance": 0.009040186181664467, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.055741358548402786, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.46596676111221313, - "samples_ms": [ - 0.5198419094085693, - 0.46596676111221313, - 0.4613921046257019 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2003796398639679, - "samples_ms": [ - 0.20474568009376526, - 0.19324570894241333, - 0.2003796398639679 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.19286200404167175, - "samples_ms": [ - 0.20584836602210999, - 0.19286200404167175, - 0.18569082021713257 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.548110067844391, - "samples_ms": [ - 4.740048199892044, - 4.08400222659111, - 4.548110067844391 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.10885298252105713, - "samples_ms": [ - 0.10885298252105713, - 0.09490549564361572, - 0.25159865617752075 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.271624118089676, - "samples_ms": [ - 1.271624118089676, - 1.352418214082718, - 1.170370727777481 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01564645767211914, - "max_error_over_tolerance": 0.12006834149360657, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625476837158203, - "max_error_over_tolerance": 0.08660759031772614, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6065703928470612, - "samples_ms": [ - 0.6653815507888794, - 0.6065703928470612, - 0.5939602851867676 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.28814375400543213, - "samples_ms": [ - 0.28814375400543213, - 0.26183947920799255, - 0.29589608311653137 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 4161024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.32318755984306335, - "samples_ms": [ - 0.34091994166374207, - 0.31236931681632996, - 0.32318755984306335 - ], - "peak_allocated_bytes": 390986240, - "incremental_peak_bytes": 160946688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.4117773175239563, - "samples_ms": [ - 3.8426965475082397, - 3.3334828913211823, - 3.4117773175239563 - ], - "peak_allocated_bytes": 564981248, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1622922718524933, - "samples_ms": [ - 0.1622922718524933, - 0.14113634824752808, - 0.18216297030448914 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.8786080181598663, - "samples_ms": [ - 2.4149343371391296, - 1.8786080181598663, - 1.6277916729450226 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 1, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.006100177764892578, - "max_error_over_tolerance": 0.067171111702919, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0007508695125579834, - "max_error_over_tolerance": 0.0071157063357532024, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.05316777154803276, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6371252238750458, - "samples_ms": [ - 0.6371252238750458, - 0.5845017731189728, - 0.7134564220905304 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.24210289120674133, - "samples_ms": [ - 0.2524815499782562, - 0.24210289120674133, - 0.23226439952850342 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2515725791454315, - "samples_ms": [ - 0.2801157534122467, - 0.2515725791454315, - 0.2417340874671936 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.0545853078365326, - "samples_ms": [ - 3.012627363204956, - 3.1444616615772247, - 3.0545853078365326 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1009032130241394, - "samples_ms": [ - 0.10890886187553406, - 0.1009032130241394, - 0.0995248556137085 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.6249621510505676, - "samples_ms": [ - 0.841282308101654, - 0.6249621510505676, - 0.4223547875881195 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007819890975952148, - "max_error_over_tolerance": 0.08643733710050583, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009521245956420898, - "max_error_over_tolerance": 0.009015905670821667, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014435768127441406, - "max_error_over_tolerance": 0.07634487003087997, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6174556910991669, - "samples_ms": [ - 0.528968870639801, - 0.6931237876415253, - 0.6174556910991669 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.215090811252594, - "samples_ms": [ - 0.22701174020767212, - 0.215090811252594, - 0.21107494831085205 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.21208450198173523, - "samples_ms": [ - 0.22354722023010254, - 0.21208450198173523, - 0.20722299814224243 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 2.8594695031642914, - "samples_ms": [ - 3.271866589784622, - 2.7009136974811554, - 2.8594695031642914 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08808448910713196, - "samples_ms": [ - 0.10520219802856445, - 0.08808448910713196, - 0.08613988757133484 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.49784407019615173, - "samples_ms": [ - 0.7374957203865051, - 0.49784407019615173, - 0.3265179693698883 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078012943267822266, - "max_error_over_tolerance": 0.08598456531763077, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009499192237854004, - "max_error_over_tolerance": 0.009035039693117142, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015491485595703125, - "max_error_over_tolerance": 0.08537446707487106, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6163567304611206, - "samples_ms": [ - 0.6814412772655487, - 0.6163567304611206, - 0.5919598042964935 - ], - "peak_allocated_bytes": 231888896, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2610310912132263, - "samples_ms": [ - 0.26984885334968567, - 0.25866180658340454, - 0.2610310912132263 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.32006576657295227, - "samples_ms": [ - 0.370122492313385, - 0.32006576657295227, - 0.31005963683128357 - ], - "peak_allocated_bytes": 391910912, - "incremental_peak_bytes": 161409024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.758283495903015, - "samples_ms": [ - 4.77876141667366, - 4.7095417976379395, - 4.758283495903015 - ], - "peak_allocated_bytes": 565443584, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1256018877029419, - "samples_ms": [ - 0.13486668467521667, - 0.1256018877029419, - 0.1223050057888031 - ], - "peak_allocated_bytes": 231888896, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.401267945766449, - "samples_ms": [ - 1.401267945766449, - 1.3131499290466309, - 1.4289245009422302 - ], - "peak_allocated_bytes": 389728256, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 1, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.52587890625e-05, - "max_error_over_tolerance": 0.00030462356517091393, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0004891455173492432, - "max_error_over_tolerance": 0.0047654458321630955, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.03244646266102791, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5212686955928802, - "samples_ms": [ - 0.5578920245170593, - 0.5212686955928802, - 0.5110912024974823 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.5053989589214325, - "samples_ms": [ - 0.49149245023727417, - 0.6247162818908691, - 0.5053989589214325 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.28608739376068115, - "samples_ms": [ - 0.37166476249694824, - 0.28608739376068115, - 0.26038289070129395 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 2.69177183508873, - "samples_ms": [ - 2.622220665216446, - 2.69177183508873, - 2.7831755578517914 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08979439735412598, - "samples_ms": [ - 0.09759515523910522, - 0.08979439735412598, - 0.08885189890861511 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.7829070091247559, - "samples_ms": [ - 0.7398799061775208, - 0.7829070091247559, - 0.828668475151062 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.527508720755577e-05, - "max_error_over_tolerance": 0.00030498611158691347, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000972449779510498, - "max_error_over_tolerance": 0.009215136989951134, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.055741358548402786, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.620037317276001, - "samples_ms": [ - 0.6789267063140869, - 0.620037317276001, - 0.599246472120285 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2673007547855377, - "samples_ms": [ - 0.2936124801635742, - 0.2673007547855377, - 0.2442002296447754 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.3009811043739319, - "samples_ms": [ - 0.3041587769985199, - 0.2967715263366699, - 0.3009811043739319 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 7.314201444387436, - "samples_ms": [ - 9.714469313621521, - 6.630800664424896, - 7.314201444387436 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11700019240379333, - "samples_ms": [ - 0.11700019240379333, - 0.11009350419044495, - 0.11860206723213196 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.337241381406784, - "samples_ms": [ - 1.394171267747879, - 1.3254545629024506, - 1.337241381406784 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "cancellation", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 1.5273690223693848e-05, - "max_error_over_tolerance": 0.0003049805236514658, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009714066982269287, - "max_error_over_tolerance": 0.009242905303835869, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625, - "max_error_over_tolerance": 0.08660212904214859, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6686709821224213, - "samples_ms": [ - 0.7194802165031433, - 0.6686709821224213, - 0.6463862955570221 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.29221922159194946, - "samples_ms": [ - 0.322144478559494, - 0.29221922159194946, - 0.2859048545360565 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 4161024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.3761462867259979, - "samples_ms": [ - 0.3761462867259979, - 0.3552660346031189, - 0.3951750695705414 - ], - "peak_allocated_bytes": 390986240, - "incremental_peak_bytes": 160946688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.301741719245911, - "samples_ms": [ - 6.697036325931549, - 4.301741719245911, - 3.891546279191971 - ], - "peak_allocated_bytes": 564981248, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11081993579864502, - "samples_ms": [ - 0.11987611651420593, - 0.11081993579864502, - 0.1092180609703064 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.0448917746543884, - "samples_ms": [ - 1.2847892940044403, - 1.0448917746543884, - 0.5874298512935638 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "checkpoint_fixture", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12514114379882812, - "max_error_over_tolerance": 0.17978119850158691, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015633583068847656, - "max_error_over_tolerance": 0.08594760298728943, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015613555908203125, - "max_error_over_tolerance": 0.08570058643817902, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.46795979142189026, - "samples_ms": [ - 1.3030916452407837, - 0.46795979142189026, - 0.4540979862213135 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.3589130938053131, - "samples_ms": [ - 0.2717040479183197, - 0.37369877099990845, - 0.3589130938053131 - ], - "peak_allocated_bytes": 233738240, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.41437894105911255, - "samples_ms": [ - 0.6911270320415497, - 0.41437894105911255, - 0.23768097162246704 - ], - "peak_allocated_bytes": 390488064, - "incremental_peak_bytes": 160448512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.3845027685165405, - "samples_ms": [ - 5.283676087856293, - 4.364609718322754, - 4.3845027685165405 - ], - "peak_allocated_bytes": 563594240, - "incremental_peak_bytes": 333554688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11146068572998047, - "samples_ms": [ - 0.12151896953582764, - 0.11146068572998047, - 0.1089051365852356 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.294311136007309, - "samples_ms": [ - 1.3019554316997528, - 1.2869052588939667, - 1.294311136007309 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "69ae1e8ba2921e3feb7cc064a1ab33eb0f5546b32ac127b6d8054d4175c72021", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - } - ], - "acceptance_complete": false, - "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", - "provenance": { - "git_commit": "32b765ec992cd1206517104ec66506881203c91c", - "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", - "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", - "python": "3.10.12", - "torch": "2.6.0+cu124", - "torch_cuda": "12.4", - "torch_hip": null, - "contract_version": "ws1-c1-v2", - "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "args": { - "rows": "1,33,129", - "families": "random,swiglu_like,cancellation", - "repeats": 3, - "seed": 420, - "output": "artifacts/h3-down/h100-fp32-cold-repeat/report.json", - "build_log": null, - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "compare_vllm": false, - "fixture": "${FIXTURE_ROOT}/transformer-block0-down.pt" - }, - "environment": { - "KERNEL_ALIGN_USE_FAST_MATH": null, - "KERNEL_ALIGN_DET_GEMM_SM90": null, - "RL_KERNEL_DET_GEMM_BACKEND": null, - "CUBLAS_WORKSPACE_CONFIG": null, - "CUBLASLT_WORKSPACE_SIZE": null, - "TORCH_CUDA_ARCH_LIST": null - }, - "source_hashes": { - "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", - "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", - "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", - "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", - "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", - "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", - "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", - "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", - "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" - }, - "build_flags_verified_at_runtime": false, - "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", - "device": "NVIDIA H100 80GB HBM3", - "capability": [ - 9, - 0 - ], - "candidate_tf32_enabled": false, - "reference_tf32_enabled": false, - "torch_baseline_bf16_reduced_precision_reduction": false, - "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" - }, - "fixture": { - "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", - "metadata": { - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "model_variant": "transformer", - "weight_key": "transformer_blocks.0.ff.net.2.weight", - "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", - "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", - "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", - "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", - "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", - "activation_kind": "synthetic", - "seed": 420 - } - }, - "remaining_acceptance": [ - "maintainer review of arithmetic contract and covered matrix", - "independent build/environment repeat and prior-art performance decision", - "real captured SwiGLU activations" - ] - }, - "triton_3_4_0": { - "schema": "h3-down-qualification-v2", - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "status": "passed", - "cases": [ - { - "rows": 1, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0077440738677978516, - "max_error_over_tolerance": 0.08342820405960083, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000848233699798584, - "max_error_over_tolerance": 0.008046787232160568, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.0324886292219162, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.9755549728870392, - "samples_ms": [ - 0.756826251745224, - 2.429340034723282, - 2.3806132376194, - 1.9755549728870392, - 1.5296190977096558 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.8079968392848969, - "samples_ms": [ - 0.9192898869514465, - 0.867113471031189, - 0.7740408182144165, - 0.8079968392848969, - 0.723525881767273 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.27946382761001587, - "samples_ms": [ - 0.3366135060787201, - 0.2865418791770935, - 0.2709105610847473, - 0.272214412689209, - 0.27946382761001587 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.154114216566086, - "samples_ms": [ - 5.06950169801712, - 5.154114216566086, - 5.122512578964233, - 5.297519266605377, - 5.531996488571167 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08181482553482056, - "samples_ms": [ - 0.10541826486587524, - 0.08444488048553467, - 0.08181482553482056, - 0.0813119113445282, - 0.08007138967514038 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.8751248717308044, - "samples_ms": [ - 1.8434450030326843, - 2.448052167892456, - 2.0304471254348755, - 1.5846826136112213, - 1.8751248717308044 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "442ea5185d0fd78f99a7d9e7bfc6024673a950fa98f4315816aa22c4a38c5053", - "cubin_sha256": "0be9cc59dade284d97c8c7831a3e463e5e96b8c3dcb63cab3962c9ea3141163d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "71a486a14ced26068c5e89b957e640da2b1ce1472c1bab9830d521e55fd5de55", - "cubin_sha256": "3de63ceb61e4bcaf96a4ce172f63ce70126fb67774c1c6a56c5dd22a489ccfda", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06", - "cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "vllm_comparison": { - "status": "passed_probe", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0077440738677978516, - "max_error_over_tolerance": 0.08342820405960083, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000848233699798584, - "max_error_over_tolerance": 0.008046787232160568, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.0324886292219162, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton" - } - ], - "benchmarks": { - "forward": { - "median_ms": 0.35585835576057434, - "samples_ms": [ - 0.39868801832199097, - 0.35585835576057434, - 0.36633387207984924, - 0.33082813024520874, - 0.32956525683403015 - ], - "peak_allocated_bytes": 838020608, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dx": { - "median_ms": 0.3727339208126068, - "samples_ms": [ - 0.42771175503730774, - 0.38250163197517395, - 0.3697015345096588, - 0.3727339208126068, - 0.3604479134082794 - ], - "peak_allocated_bytes": 838060032, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dw": { - "median_ms": 0.23178011178970337, - "samples_ms": [ - 0.28607621788978577, - 0.25910884141921997, - 0.22685527801513672, - 0.23178011178970337, - 0.2170279622077942 - ], - "peak_allocated_bytes": 992207872, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", - "triton_version": "3.4.0", - "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", - "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", - "configs": [ - { - "shape": { - "M": 1, - "N": 5376, - "K": 14336 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 1, - "N": 14336, - "K": 5376 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 5376, - "N": 14336, - "K": 1 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - } - ], - "qualification": "partial_prior_art_probe_only" - }, - "passed": true - }, - { - "rows": 129, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01564645767211914, - "max_error_over_tolerance": 0.12006834149360657, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625476837158203, - "max_error_over_tolerance": 0.08660759031772614, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5118362605571747, - "samples_ms": [ - 0.7387325167655945, - 1.6151517629623413, - 0.48866868019104004, - 0.5118362605571747, - 0.48043206334114075 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.20536035299301147, - "samples_ms": [ - 0.22369623184204102, - 0.2122148871421814, - 0.20536035299301147, - 0.2004280686378479, - 0.20079687237739563 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 4161024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.23826584219932556, - "samples_ms": [ - 0.2539418637752533, - 0.23826584219932556, - 0.23221969604492188, - 0.2310052514076233, - 0.24832412600517273 - ], - "peak_allocated_bytes": 390986240, - "incremental_peak_bytes": 160946688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.4094473123550415, - "samples_ms": [ - 4.275541752576828, - 4.517361521720886, - 4.4094473123550415, - 4.568081349134445, - 4.234585911035538 - ], - "peak_allocated_bytes": 564981248, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.12133270502090454, - "samples_ms": [ - 0.1527555286884308, - 0.12133270502090454, - 0.12947991490364075, - 0.12053549289703369, - 0.11564046144485474 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.3994351029396057, - "samples_ms": [ - 1.2967772781848907, - 1.314643770456314, - 1.3994351029396057, - 1.4288835227489471, - 1.4533214271068573 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "b8fc72a5623290859b63cc1099242078498d7aa6da0a319c35c9e6026617f553", - "cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "999ed0c8d411455da93d5dd63f766ec664ac762855dd057d45edad2325e0801d", - "cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06", - "cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "vllm_comparison": { - "status": "passed_probe", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01564645767211914, - "max_error_over_tolerance": 0.12006834149360657, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009601116180419922, - "max_error_over_tolerance": 0.009125219658017159, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015625476837158203, - "max_error_over_tolerance": 0.08660759031772614, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton" - } - ], - "benchmarks": { - "forward": { - "median_ms": 0.36650896072387695, - "samples_ms": [ - 0.4074461758136749, - 0.3713257610797882, - 0.36650896072387695, - 0.36140158772468567, - 0.3532879054546356 - ], - "peak_allocated_bytes": 865554944, - "incremental_peak_bytes": 2311680, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dx": { - "median_ms": 0.4915297031402588, - "samples_ms": [ - 0.5265325307846069, - 0.4915297031402588, - 0.4705600440502167, - 0.4768744111061096, - 0.6239041686058044 - ], - "peak_allocated_bytes": 869264384, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dw": { - "median_ms": 0.3756880760192871, - "samples_ms": [ - 0.4069618880748749, - 0.3756880760192871, - 0.36858394742012024, - 0.3760494291782379, - 0.3723315894603729 - ], - "peak_allocated_bytes": 1021111296, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", - "triton_version": "3.4.0", - "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", - "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", - "configs": [ - { - "shape": { - "M": 129, - "N": 5376, - "K": 14336 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 129, - "N": 14336, - "K": 5376 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 5376, - "N": 14336, - "K": 129 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - } - ], - "qualification": "partial_prior_art_probe_only" - }, - "passed": true - }, - { - "rows": 8192, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015656471252441406, - "max_error_over_tolerance": 0.11954245716333389, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000977635383605957, - "max_error_over_tolerance": 0.009301135316491127, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12531661987304688, - "max_error_over_tolerance": 0.1686829924583435, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 7.39080086350441, - "samples_ms": [ - 7.39080086350441, - 7.132500410079956, - 6.938610225915909, - 7.768817245960236, - 8.397072553634644 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 4.552166908979416, - "samples_ms": [ - 4.7802068293094635, - 4.4778212904930115, - 4.534110426902771, - 4.614364355802536, - 4.552166908979416 - ], - "peak_allocated_bytes": 1014235136, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 4.351019859313965, - "samples_ms": [ - 4.253793507814407, - 4.231806844472885, - 4.351019859313965, - 4.374224692583084, - 4.366464912891388 - ], - "peak_allocated_bytes": 933494784, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 19.10606399178505, - "samples_ms": [ - 22.746942937374115, - 19.242648035287857, - 18.63836869597435, - 18.933270126581192, - 19.10606399178505 - ], - "peak_allocated_bytes": 2291867648, - "incremental_peak_bytes": 1512513536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 1.7073601484298706, - "samples_ms": [ - 1.7186403274536133, - 1.7073601484298706, - 1.7057321965694427, - 1.7052367329597473, - 1.7091743648052216 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 5.997553467750549, - "samples_ms": [ - 5.202878266572952, - 5.3045786917209625, - 5.997553467750549, - 9.920638054609299, - 8.24371725320816 - ], - "peak_allocated_bytes": 1256456192, - "incremental_peak_bytes": 477102080, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "96b3410b50e0ebcc24825af8569468d59d46882af5be72b7bfd8ee36bad4f2c2", - "cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105", - "compiled_num_warps": 4, - "grid": [ - 128, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "bb8d38510563eb1678e3e76afe4e649f1447d27e9734ccd6ef2eb116502c5d58", - "cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64", - "compiled_num_warps": 4, - "grid": [ - 128, - 224 - ] - }, - "canonical_active_rows": 8192, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06", - "cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 5376 - ] - }, - "dx": { - "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 14336 - ] - }, - "dw": { - "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "vllm_comparison": { - "status": "passed_probe", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015656471252441406, - "max_error_over_tolerance": 0.11954245716333389, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000977635383605957, - "max_error_over_tolerance": 0.009301135316491127, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12531661987304688, - "max_error_over_tolerance": 0.1686829924583435, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton" - } - ], - "benchmarks": { - "forward": { - "median_ms": 2.499658614397049, - "samples_ms": [ - 2.5562308728694916, - 2.5168806314468384, - 2.499658614397049, - 2.48013436794281, - 2.489827573299408 - ], - "peak_allocated_bytes": 2540830720, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dx": { - "median_ms": 5.712740123271942, - "samples_ms": [ - 5.7429783046245575, - 5.476012825965881, - 5.496017634868622, - 5.7668983936309814, - 5.712740123271942 - ], - "peak_allocated_bytes": 2775722496, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dw": { - "median_ms": 10.247066617012024, - "samples_ms": [ - 10.233435779809952, - 10.858424007892609, - 10.247066617012024, - 10.259285569190979, - 10.230351239442825 - ], - "peak_allocated_bytes": 2841800704, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", - "triton_version": "3.4.0", - "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", - "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", - "configs": [ - { - "shape": { - "M": 8192, - "N": 5376, - "K": 14336 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 8192, - "N": 14336, - "K": 5376 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 5376, - "N": 14336, - "K": 8192 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - } - ], - "qualification": "partial_prior_art_probe_only" - }, - "passed": true - }, - { - "rows": 32768, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01569652557373047, - "max_error_over_tolerance": 0.12013405561447144, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781122207641602, - "max_error_over_tolerance": 0.009311916306614876, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.2525482177734375, - "max_error_over_tolerance": 0.18232759833335876, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 41.59632697701454, - "samples_ms": [ - 41.687577962875366, - 41.54922440648079, - 41.800860315561295, - 41.590429842472076, - 41.59632697701454 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 31.687993556261063, - "samples_ms": [ - 29.00845929980278, - 24.406280368566513, - 33.68991240859032, - 31.687993556261063, - 31.89874067902565 - ], - "peak_allocated_bytes": 3393191936, - "incremental_peak_bytes": 939524096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 20.97996324300766, - "samples_ms": [ - 19.690733402967453, - 21.237235516309738, - 20.029544830322266, - 22.116035223007202, - 20.97996324300766 - ], - "peak_allocated_bytes": 2607808512, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 99.08927977085114, - "samples_ms": [ - 97.93517366051674, - 99.08927977085114, - 107.6027862727642, - 96.2594524025917, - 99.96020048856735 - ], - "peak_allocated_bytes": 7578877952, - "incremental_peak_bytes": 5125210112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 6.319291889667511, - "samples_ms": [ - 6.33331760764122, - 6.319291889667511, - 6.296105682849884, - 6.304707378149033, - 7.043942809104919 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 20.878225564956665, - "samples_ms": [ - 25.30929446220398, - 21.993432193994522, - 20.878225564956665, - 20.874205976724625, - 20.87520807981491 - ], - "peak_allocated_bytes": 3899654144, - "incremental_peak_bytes": 1445986304, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "96b3410b50e0ebcc24825af8569468d59d46882af5be72b7bfd8ee36bad4f2c2", - "cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105", - "compiled_num_warps": 4, - "grid": [ - 512, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "bb8d38510563eb1678e3e76afe4e649f1447d27e9734ccd6ef2eb116502c5d58", - "cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64", - "compiled_num_warps": 4, - "grid": [ - 512, - 224 - ] - }, - "canonical_active_rows": 32768, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06", - "cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 5376 - ] - }, - "dx": { - "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 14336 - ] - }, - "dw": { - "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "vllm_comparison": { - "status": "passed_probe", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01569652557373047, - "max_error_over_tolerance": 0.12013405561447144, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009781122207641602, - "max_error_over_tolerance": 0.009311916306614876, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.2525482177734375, - "max_error_over_tolerance": 0.18232759833335876, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton" - } - ], - "benchmarks": { - "forward": { - "median_ms": 9.66041162610054, - "samples_ms": [ - 9.503252804279327, - 9.553678333759308, - 10.058004409074783, - 9.66041162610054, - 9.696289896965027 - ], - "peak_allocated_bytes": 7649886208, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dx": { - "median_ms": 21.029628813266754, - "samples_ms": [ - 21.116141229867935, - 21.001089364290237, - 20.985208451747894, - 21.029628813266754, - 21.04152739048004 - ], - "peak_allocated_bytes": 8589421056, - "incremental_peak_bytes": 939524096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dw": { - "median_ms": 40.02102091908455, - "samples_ms": [ - 40.06137698888779, - 40.02102091908455, - 40.013208985328674, - 39.991073310375214, - 40.18115624785423 - ], - "peak_allocated_bytes": 8391258112, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", - "triton_version": "3.4.0", - "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", - "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", - "configs": [ - { - "shape": { - "M": 32768, - "N": 5376, - "K": 14336 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 32768, - "N": 14336, - "K": 5376 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 5376, - "N": 14336, - "K": 32768 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - } - ], - "qualification": "partial_prior_art_probe_only" - }, - "passed": true - }, - { - "rows": 129, - "input_family": "checkpoint_fixture", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12514114379882812, - "max_error_over_tolerance": 0.17978119850158691, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015633583068847656, - "max_error_over_tolerance": 0.08594760298728943, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015613555908203125, - "max_error_over_tolerance": 0.08570058643817902, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.4480406641960144, - "samples_ms": [ - 0.5202442407608032, - 0.46155601739883423, - 0.4480406641960144, - 0.44644251465797424, - 0.4439912736415863 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19307062029838562, - "samples_ms": [ - 0.20404160022735596, - 0.20174309611320496, - 0.18886849284172058, - 0.19307062029838562, - 0.1902841031551361 - ], - "peak_allocated_bytes": 233738240, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.22620707750320435, - "samples_ms": [ - 0.2653747797012329, - 0.22744014859199524, - 0.22080540657043457, - 0.21801888942718506, - 0.22620707750320435 - ], - "peak_allocated_bytes": 390488064, - "incremental_peak_bytes": 160448512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.0629821121692657, - "samples_ms": [ - 2.863481640815735, - 3.0629821121692657, - 3.391604870557785, - 3.008224070072174, - 3.2156631350517273 - ], - "peak_allocated_bytes": 563594240, - "incremental_peak_bytes": 333554688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1094229519367218, - "samples_ms": [ - 0.12498721480369568, - 0.1124851405620575, - 0.1094229519367218, - 0.10900571942329407, - 0.1081712543964386 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.41426345705986023, - "samples_ms": [ - 0.5338490009307861, - 0.3857165575027466, - 0.5918629467487335, - 0.41426345705986023, - 0.36514177918434143 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "b8fc72a5623290859b63cc1099242078498d7aa6da0a319c35c9e6026617f553", - "cubin_sha256": "3769a5a2630bb93c1da7d969fa5385042225cf59923cc32617186324c354e105", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "999ed0c8d411455da93d5dd63f766ec664ac762855dd057d45edad2325e0801d", - "cubin_sha256": "a77f6258fb19eed4959cc6363b2fc6ab6a83889ad8052432664f9560a0aded64", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.4.0", - "compiled_kernel_hash": "717fc47570df3c32829523a94e215d1970db9027f9be6f327ddbca62fe05df06", - "cubin_sha256": "f78e5ef03f8e4b3a2a4698e4a6e383147601fbf51beac0eed8766b60af9a24c2", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "vllm_comparison": { - "status": "passed_probe", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12514114379882812, - "max_error_over_tolerance": 0.17978119850158691, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015633583068847656, - "max_error_over_tolerance": 0.08594760298728943, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015613555908203125, - "max_error_over_tolerance": 0.08570058643817902, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton" - } - ], - "benchmarks": { - "forward": { - "median_ms": 0.31507760286331177, - "samples_ms": [ - 0.33548474311828613, - 0.32701343297958374, - 0.31507760286331177, - 0.3068968653678894, - 0.30349940061569214 - ], - "peak_allocated_bytes": 865554944, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dx": { - "median_ms": 0.41652098298072815, - "samples_ms": [ - 0.4444718360900879, - 0.41950494050979614, - 0.41652098298072815, - 0.3989376127719879, - 0.397317111492157 - ], - "peak_allocated_bytes": 869264384, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "dw": { - "median_ms": 0.32254308462142944, - "samples_ms": [ - 0.5048699676990509, - 0.47458335757255554, - 0.32254308462142944, - 0.31186267733573914, - 0.30764564871788025 - ], - "peak_allocated_bytes": 1022035968, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "source_commit": "7797b6022c129b862e45ae6aed08822e65d1bccb", - "triton_version": "3.4.0", - "source_sha256": "3c53d85dca812c082f051eff6fbcdad31ffd140982db9b68929ec9f1a95f2bf1", - "configs_sha256": "f1c1b65a2a65054903be5fdd5432b73a340faf87bf32ca024124efff0abb50e2", - "configs": [ - { - "shape": { - "M": 129, - "N": 5376, - "K": 14336 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 129, - "N": 14336, - "K": 5376 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - }, - { - "shape": { - "M": 5376, - "N": 14336, - "K": 129 - }, - "dtype": "bfloat16", - "config": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 8, - "num_stages": 3, - "num_warps": 8 - }, - "selection": "default", - "is_tuned": false, - "device_capability": [ - 9, - 0 - ], - "arch_family": "hopper" - } - ], - "qualification": "partial_prior_art_probe_only" - }, - "passed": true - } - ], - "acceptance_complete": false, - "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", - "provenance": { - "git_commit": "32b765ec992cd1206517104ec66506881203c91c", - "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", - "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", - "python": "3.10.12", - "torch": "2.6.0+cu124", - "torch_cuda": "12.4", - "torch_hip": null, - "contract_version": "ws1-c1-v2", - "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "args": { - "rows": "1,129,8192,32768", - "families": "random", - "repeats": 5, - "seed": 420, - "output": "artifacts/h3-down/h100-triton34-comparison/report.json", - "build_log": null, - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "compare_vllm": true, - "fixture": "${FIXTURE_ROOT}/transformer-block0-down.pt" - }, - "environment": { - "KERNEL_ALIGN_USE_FAST_MATH": null, - "KERNEL_ALIGN_DET_GEMM_SM90": null, - "RL_KERNEL_DET_GEMM_BACKEND": null, - "CUBLAS_WORKSPACE_CONFIG": null, - "CUBLASLT_WORKSPACE_SIZE": null, - "TORCH_CUDA_ARCH_LIST": null - }, - "source_hashes": { - "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", - "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", - "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", - "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", - "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", - "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", - "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", - "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", - "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" - }, - "build_flags_verified_at_runtime": false, - "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", - "device": "NVIDIA H100 80GB HBM3", - "capability": [ - 9, - 0 - ], - "candidate_tf32_enabled": false, - "reference_tf32_enabled": false, - "torch_baseline_bf16_reduced_precision_reduction": false, - "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" - }, - "fixture": { - "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", - "metadata": { - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "model_variant": "transformer", - "weight_key": "transformer_blocks.0.ff.net.2.weight", - "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", - "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", - "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", - "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", - "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", - "activation_kind": "synthetic", - "seed": 420 - } - }, - "remaining_acceptance": [ - "maintainer review of arithmetic contract and covered matrix", - "independent build/environment repeat and prior-art performance decision", - "real captured SwiGLU activations" - ] - }, - "cold_cache_output_comparison": { - "status": "passed", - "scope": "Same H100 and PyTorch profile; separate process and fresh Triton 3.2.0 cache; not another GPU or runtime environment", - "checks": [ - { - "input_family": "random", - "rows": 1, - "tensor": "y", - "passed": true, - "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386" - }, - { - "input_family": "random", - "rows": 1, - "tensor": "dx", - "passed": true, - "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f" - }, - { - "input_family": "random", - "rows": 1, - "tensor": "dw", - "passed": true, - "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9" - }, - { - "input_family": "random", - "rows": 33, - "tensor": "y", - "passed": true, - "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869" - }, - { - "input_family": "random", - "rows": 33, - "tensor": "dx", - "passed": true, - "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee" - }, - { - "input_family": "random", - "rows": 33, - "tensor": "dw", - "passed": true, - "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078" - }, - { - "input_family": "swiglu_like", - "rows": 1, - "tensor": "y", - "passed": true, - "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb" - }, - { - "input_family": "swiglu_like", - "rows": 1, - "tensor": "dx", - "passed": true, - "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43" - }, - { - "input_family": "swiglu_like", - "rows": 1, - "tensor": "dw", - "passed": true, - "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda" - }, - { - "input_family": "swiglu_like", - "rows": 33, - "tensor": "y", - "passed": true, - "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e" - }, - { - "input_family": "swiglu_like", - "rows": 33, - "tensor": "dx", - "passed": true, - "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60" - }, - { - "input_family": "swiglu_like", - "rows": 33, - "tensor": "dw", - "passed": true, - "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f" - }, - { - "input_family": "swiglu_like", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236" - }, - { - "input_family": "swiglu_like", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8" - }, - { - "input_family": "swiglu_like", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f" - }, - { - "input_family": "cancellation", - "rows": 1, - "tensor": "y", - "passed": true, - "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421" - }, - { - "input_family": "cancellation", - "rows": 1, - "tensor": "dx", - "passed": true, - "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730" - }, - { - "input_family": "cancellation", - "rows": 1, - "tensor": "dw", - "passed": true, - "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3" - }, - { - "input_family": "cancellation", - "rows": 33, - "tensor": "y", - "passed": true, - "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b" - }, - { - "input_family": "cancellation", - "rows": 33, - "tensor": "dx", - "passed": true, - "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3" - }, - { - "input_family": "cancellation", - "rows": 33, - "tensor": "dw", - "passed": true, - "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18" - }, - { - "input_family": "cancellation", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9" - }, - { - "input_family": "cancellation", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930" - }, - { - "input_family": "cancellation", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41" - } - ] - }, - "compiler_version_output_comparison": { - "status": "passed", - "scope": "Same H100/PyTorch; Triton 3.2.0 versus separately pinned 3.4.0 using independent JIT caches; selected cases only", - "checks": [ - { - "input_family": "random", - "rows": 1, - "tensor": "y", - "passed": true, - "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386" - }, - { - "input_family": "random", - "rows": 1, - "tensor": "dx", - "passed": true, - "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f" - }, - { - "input_family": "random", - "rows": 1, - "tensor": "dw", - "passed": true, - "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d" - }, - { - "input_family": "random", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078" - }, - { - "input_family": "random", - "rows": 8192, - "tensor": "y", - "passed": true, - "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8" - }, - { - "input_family": "random", - "rows": 8192, - "tensor": "dx", - "passed": true, - "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66" - }, - { - "input_family": "random", - "rows": 8192, - "tensor": "dw", - "passed": true, - "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328" - }, - { - "input_family": "random", - "rows": 32768, - "tensor": "y", - "passed": true, - "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949" - }, - { - "input_family": "random", - "rows": 32768, - "tensor": "dx", - "passed": true, - "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515" - }, - { - "input_family": "random", - "rows": 32768, - "tensor": "dw", - "passed": true, - "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "y", - "passed": true, - "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "dx", - "passed": true, - "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84" - }, - { - "input_family": "checkpoint_fixture", - "rows": 129, - "tensor": "dw", - "passed": true, - "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41" - } - ] - } - } - } - } -} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json b/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json deleted file mode 100644 index 68f7e8efd..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json +++ /dev/null @@ -1,3112 +0,0 @@ -{ - "schema_version": 1, - "gpu": "h100", - "provenance": { - "derivative": "Sanitized historical evidence; only execution paths replaced with portable placeholders. All measurements and findings retained. No new GPU run.", - "original_commit": "006021ca9875cdc4ae66af7ed184514070f3274a", - "original_path": "docs/operators/evidence/h3-down-h100-20261010/h100-summary.json" - }, - "artifacts": { - "h100-summary.json": { - "original_sha256": "62180d60270b40c4857dc78f8f85c9e782f5357752e9adb78144f348cc256d07", - "format": "json", - "content": { - "schema": "h3-down-evidence-summary-v1", - "readiness": "ready_for_operator_review", - "production_qualified": false, - "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", - "full_report_header": { - "schema": "h3-down-qualification-v2", - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "status": "passed", - "acceptance_complete": false, - "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", - "provenance": { - "git_commit": "32b765ec992cd1206517104ec66506881203c91c", - "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_fp32_backend.py\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/prepare_h3_down_fixture.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_fixture.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py", - "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", - "python": "3.10.12", - "torch": "2.6.0+cu124", - "torch_cuda": "12.4", - "torch_hip": null, - "contract_version": "ws1-c1-v2", - "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "args": { - "rows": "1,31,32,33,127,128,129,256,513,1024,2048,4096,8192,16385,24577,32768", - "families": "random,swiglu_like,cancellation", - "repeats": 5, - "seed": 420, - "output": "artifacts/h3-down/h100-fp32-full/report.json", - "build_log": null, - "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "compare_vllm": false, - "fixture": "${FIXTURE_ROOT}/transformer-block0-down.pt" - }, - "environment": { - "KERNEL_ALIGN_USE_FAST_MATH": null, - "KERNEL_ALIGN_DET_GEMM_SM90": null, - "RL_KERNEL_DET_GEMM_BACKEND": null, - "CUBLAS_WORKSPACE_CONFIG": null, - "CUBLASLT_WORKSPACE_SIZE": null, - "TORCH_CUDA_ARCH_LIST": null - }, - "source_hashes": { - "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", - "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", - "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", - "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", - "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", - "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", - "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", - "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", - "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" - }, - "build_flags_verified_at_runtime": false, - "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", - "device": "NVIDIA H100 80GB HBM3", - "capability": [ - 9, - 0 - ], - "candidate_tf32_enabled": false, - "reference_tf32_enabled": false, - "torch_baseline_bf16_reduced_precision_reduction": false, - "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" - }, - "fixture": { - "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", - "metadata": { - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "model_variant": "transformer", - "weight_key": "transformer_blocks.0.ff.net.2.weight", - "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", - "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", - "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", - "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", - "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", - "activation_kind": "synthetic", - "seed": 420 - } - }, - "remaining_acceptance": [ - "maintainer review of arithmetic contract and covered matrix", - "independent build/environment repeat and prior-art performance decision", - "real captured SwiGLU activations" - ] - }, - "fixture_case_indexes": [ - 48 - ], - "fixture_cases": [ - { - "rows": 129, - "input_family": "checkpoint_fixture", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12514114379882812, - "max_error_over_tolerance": 0.17978119850158691, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015633583068847656, - "max_error_over_tolerance": 0.08594760298728943, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015613555908203125, - "max_error_over_tolerance": 0.08570058643817902, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.498991459608078, - "samples_ms": [ - 0.5232095718383789, - 0.498991459608078, - 0.5095936357975006, - 0.4890933632850647, - 0.48211216926574707 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.20795688033103943, - "samples_ms": [ - 0.22409483790397644, - 0.2106502652168274, - 0.2070702612400055, - 0.20523741841316223, - 0.20795688033103943 - ], - "peak_allocated_bytes": 233738240, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.2506338059902191, - "samples_ms": [ - 0.27573853731155396, - 0.2506338059902191, - 0.25071948766708374, - 0.2502724528312683, - 0.24274364113807678 - ], - "peak_allocated_bytes": 390488064, - "incremental_peak_bytes": 160448512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.032248795032501, - "samples_ms": [ - 5.024779587984085, - 4.45270910859108, - 5.032248795032501, - 5.378324538469315, - 8.565746247768402 - ], - "peak_allocated_bytes": 563594240, - "incremental_peak_bytes": 333554688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11239573359489441, - "samples_ms": [ - 0.12648478150367737, - 0.11574849486351013, - 0.11239573359489441, - 0.11062994599342346, - 0.11038035154342651 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.329515129327774, - "samples_ms": [ - 1.329515129327774, - 1.3584904372692108, - 1.4058314263820648, - 0.9705126285552979, - 0.9936131536960602 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - } - ], - "full_matrix_summary": { - "cases": 49, - "checks": 1141, - "passed_checks": 1141, - "max_error_over_tolerance": 0.18232759833335876 - }, - "fixture_provenance": { - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "model_variant": "transformer", - "weight_key": "transformer_blocks.0.ff.net.2.weight", - "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", - "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", - "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", - "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", - "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", - "activation_kind": "synthetic", - "seed": 420 - }, - "gpu_tests": { - "status": "passed", - "passed": 64, - "command": "python -m pytest -q tests/test_h3_ffn_down.py tests/test_h3_down_tooling.py tests/test_h3_down_fixture.py tests/test_tolerance_contract.py", - "original_stdout": "................................................................ [100%]\n64 passed in 16.70s\n", - "original_stdout_sha256": "89b85664fcf2fc6e0dc519e6656d0dddfe00c006ed98731ff2d9f126aba52c86" - }, - "rejected_native_candidate": { - "schema": "h3-down-qualification-v1", - "backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "status": "failed", - "cases": [ - { - "rows": 1, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.024131298065185547, - "max_error_over_tolerance": 0.3214894235134125, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.001803085207939148, - "max_error_over_tolerance": 0.017243297770619392, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00390625, - "max_error_over_tolerance": 0.0324886292219162, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "native_forward": { - "median_ms": 2.529580146074295, - "samples_ms": [ - 2.5643259286880493, - 2.473689615726471, - 2.529580146074295 - ], - "peak_allocated_bytes": 226376192, - "incremental_peak_bytes": 5057024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dx": { - "median_ms": 2.3520365357398987, - "samples_ms": [ - 2.546001225709915, - 2.3520365357398987, - 2.1655037999153137 - ], - "peak_allocated_bytes": 380534784, - "incremental_peak_bytes": 159215616, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dw_canonical_order": { - "median_ms": 1.4085769653320312, - "samples_ms": [ - 1.4320090413093567, - 1.4085769653320312, - 1.4054067432880402 - ], - "peak_allocated_bytes": 377982976, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_forward_backward": { - "median_ms": 13.572029769420624, - "samples_ms": [ - 12.907318770885468, - 13.572029769420624, - 13.592474162578583 - ], - "peak_allocated_bytes": 534715392, - "incremental_peak_bytes": 313396224, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11911988258361816, - "samples_ms": [ - 0.16237422823905945, - 0.11911988258361816, - 0.08111074566841125 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.7563531398773193, - "samples_ms": [ - 0.484466552734375, - 0.7563531398773193, - 0.7724203169345856 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "fallback": false, - "arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes", - "tile": [ - 128, - 64, - 32 - ], - "split_k": false, - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "canonical_active_rows": 1 - } - ], - "passed": true - }, - { - "rows": 33, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.03141450881958008, - "max_error_over_tolerance": 0.4475856125354767, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.002236664295196533, - "max_error_over_tolerance": 0.02126086875796318, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.01641559600830078, - "max_error_over_tolerance": 0.11125296354293823, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "native_forward": { - "median_ms": 2.881385385990143, - "samples_ms": [ - 3.069184720516205, - 2.881385385990143, - 2.5128573179244995 - ], - "peak_allocated_bytes": 228899840, - "incremental_peak_bytes": 5401088, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dx": { - "median_ms": 2.8548426926136017, - "samples_ms": [ - 2.9342174530029297, - 2.8548426926136017, - 2.647843211889267 - ], - "peak_allocated_bytes": 383631872, - "incremental_peak_bytes": 160133120, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dw_canonical_order": { - "median_ms": 1.6956888139247894, - "samples_ms": [ - 1.6956888139247894, - 1.7749592661857605, - 1.5147589147090912 - ], - "peak_allocated_bytes": 382685696, - "incremental_peak_bytes": 159186944, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_forward_backward": { - "median_ms": 11.53586059808731, - "samples_ms": [ - 11.53586059808731, - 10.692402720451355, - 14.232128858566284 - ], - "peak_allocated_bytes": 541679616, - "incremental_peak_bytes": 318180864, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.10365620255470276, - "samples_ms": [ - 0.11552125215530396, - 0.10365620255470276, - 0.1003853976726532 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.3978853821754456, - "samples_ms": [ - 1.3978853821754456, - 1.4610998332500458, - 1.0400116443634033 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "fallback": false, - "arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes", - "tile": [ - 128, - 64, - 32 - ], - "split_k": false, - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "canonical_active_rows": 33 - } - ], - "passed": true - }, - { - "rows": 129, - "input_family": "random", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.036692142486572266, - "max_error_over_tolerance": 0.519758403301239, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.002099767327308655, - "max_error_over_tolerance": 0.02042568475008011, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.040273189544677734, - "max_error_over_tolerance": 0.2996149957180023, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "native_forward": { - "median_ms": 3.5951845347881317, - "samples_ms": [ - 3.5951845347881317, - 3.9503835141658783, - 3.1119659543037415 - ], - "peak_allocated_bytes": 241597952, - "incremental_peak_bytes": 11558400, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dx": { - "median_ms": 2.6809796690940857, - "samples_ms": [ - 2.8591640293598175, - 2.6809796690940857, - 2.6392340660095215 - ], - "peak_allocated_bytes": 398491136, - "incremental_peak_bytes": 168451584, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dw_canonical_order": { - "median_ms": 2.1124370396137238, - "samples_ms": [ - 1.8919706344604492, - 3.0186958611011505, - 2.1124370396137238 - ], - "peak_allocated_bytes": 398014464, - "incremental_peak_bytes": 167974912, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_forward_backward": { - "median_ms": 11.893197894096375, - "samples_ms": [ - 10.98896935582161, - 11.893197894096375, - 12.51029223203659 - ], - "peak_allocated_bytes": 571289088, - "incremental_peak_bytes": 341249536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1161135733127594, - "samples_ms": [ - 0.1265592873096466, - 0.1161135733127594, - 0.11379271745681763 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.4508739113807678, - "samples_ms": [ - 1.4508739113807678, - 2.024903893470764, - 1.4123953878879547 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "fallback": false, - "arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes", - "tile": [ - 128, - 64, - 32 - ], - "split_k": false, - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "canonical_active_rows": 129 - } - ], - "passed": true - }, - { - "rows": 129, - "input_family": "checkpoint_fixture", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": false, - "max_abs_error": 0.40984344482421875, - "max_error_over_tolerance": 5.004410743713379, - "failing_elements": 23353, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.042763710021972656, - "max_error_over_tolerance": 0.37028035521507263, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.036380767822265625, - "max_error_over_tolerance": 0.20060789585113525, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "native_forward": { - "median_ms": 4.083797335624695, - "samples_ms": [ - 3.0049867928028107, - 6.327677518129349, - 4.083797335624695 - ], - "peak_allocated_bytes": 241519104, - "incremental_peak_bytes": 11479552, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dx": { - "median_ms": 18.319733440876007, - "samples_ms": [ - 18.319733440876007, - 14.052655547857285, - 21.322384476661682 - ], - "peak_allocated_bytes": 397971456, - "incremental_peak_bytes": 167931904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_dw_canonical_order": { - "median_ms": 3.0141323804855347, - "samples_ms": [ - 3.469545394182205, - 2.9487088322639465, - 3.0141323804855347 - ], - "peak_allocated_bytes": 396795904, - "incremental_peak_bytes": 166756352, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "native_forward_backward": { - "median_ms": 11.083446443080902, - "samples_ms": [ - 11.083446443080902, - 8.68365541100502, - 13.705644756555557 - ], - "peak_allocated_bytes": 569902080, - "incremental_peak_bytes": 339862528, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11894479393959045, - "samples_ms": [ - 0.12494996190071106, - 0.11894479393959045, - 0.11604651808738708 - ], - "peak_allocated_bytes": 231426560, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.3583170473575592, - "samples_ms": [ - 0.3861375153064728, - 0.3583170473575592, - 0.35572052001953125 - ], - "peak_allocated_bytes": 389265920, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "actual_backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", - "fallback": false, - "arithmetic": "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes", - "tile": [ - 128, - 64, - 32 - ], - "split_k": false, - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "canonical_active_rows": 129 - } - ], - "passed": false - } - ], - "acceptance_complete": false, - "scope": "one CUDA device, eager, operator-only; no TP or full-model acceptance", - "provenance": { - "git_commit": "32b765ec992cd1206517104ec66506881203c91c", - "git_status": "M .gitignore\n A docs/operators/h3-down-local-validation.md\n A docs/operators/h3-ffn-down.md\n A rl_engine/kernels/ops/h3_down_validation.py\n A rl_engine/kernels/ops/h3_ffn_down.py\n M rl_engine/kernels/registry.py\n A scripts/plot_h3_down_report.py\n A scripts/requirements-h3-down.txt\n A scripts/run_h3_down_h100.sh\n A scripts/validate_h3_ffn_down.py\n A tests/test_h3_down_tooling.py\n A tests/test_h3_ffn_down.py\n?? scripts/prepare_h3_down_fixture.py", - "git_diff_sha256": "dcbbf6328bc4324e48a0753cba99e14b0618fc673080016c575f8dd5bce2cc0e", - "python": "3.10.12", - "torch": "2.6.0+cu124", - "torch_cuda": "12.4", - "torch_hip": null, - "contract_version": "ws1-c1-v2", - "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532", - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "args": { - "rows": "1,33,129", - "families": "random", - "repeats": 3, - "seed": 420, - "output": "artifacts/h3-down/h100-smoke/report.json", - "build_log": "artifacts/h3-down/h100-smoke/build.log", - "compare_vllm": false, - "fixture": "${FIXTURE_ROOT}/transformer-block0-down.pt" - }, - "environment": { - "KERNEL_ALIGN_USE_FAST_MATH": "0", - "KERNEL_ALIGN_DET_GEMM_SM90": "1", - "RL_KERNEL_DET_GEMM_BACKEND": "sm90", - "CUBLAS_WORKSPACE_CONFIG": null, - "CUBLASLT_WORKSPACE_SIZE": null, - "TORCH_CUDA_ARCH_LIST": null - }, - "source_hashes": { - "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", - "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", - "rl_engine/kernels/ops/h3_ffn_down.py": "57916c658f451daf04380daff6404ad0924429b58df54566aae9aad575d1dec9", - "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", - "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", - "rl_engine/kernels/registry.py": "4680bf38f046487073945e0bcfb73e936f8c1183a4f5dff65d1699740c4c07e2", - "scripts/validate_h3_ffn_down.py": "6509c17e44ed1e8f5d5cabf145af4e6c59bfde47b388968f9ad26b9e797e2ec1" - }, - "build_flags_verified_at_runtime": false, - "extension_sha256": "63a935a30e8c2e71791fc7e773f8d30ab872fbee78292644e7840a71270d9fb5", - "build_log_sha256": "ce2722b632e6a0903f795d44d293c8c6c0da8a3835388da7db0d27e25eb2ef34", - "device": "NVIDIA H100 80GB HBM3", - "capability": [ - 9, - 0 - ], - "candidate_tf32_enabled": false, - "reference_tf32_enabled": false, - "torch_baseline_bf16_reduced_precision_reduction": false, - "nvidia_smi": "NVIDIA H100 80GB HBM3, GPU-260a8859-9032-5c74-50ea-0798cf7dc615, 570.211.01, 81559 MiB" - }, - "fixture": { - "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", - "metadata": { - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "model_variant": "transformer", - "weight_key": "transformer_blocks.0.ff.net.2.weight", - "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", - "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", - "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", - "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", - "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", - "activation_kind": "synthetic", - "seed": 420 - } - }, - "remaining_acceptance": [ - "maintainer review of arithmetic contract and covered matrix", - "independent build/environment repeat and prior-art performance decision", - "real captured SwiGLU activations" - ] - }, - "vllm_triton_3_2_error_log": "WARNING 10-10 14:37:52 [RL-Kernel]: Core binary extension (_C) unavailable: cannot import name '_C' from 'rl_engine' (${REPO_ROOT}/rl_engine/__init__.py). Falling back to native code.\n1 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n33 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n129 error f, context=context, options=options, codegen_fns=codegen_fns,\ntriton.compiler.errors.CompilationError: at 36:4:\n start_pid = tl.program_id(axis=0)\n num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)\n num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)\n k_tiles = tl.cdiv(K, BLOCK_SIZE_K)\n num_tiles = num_pid_m * num_pid_n\n\n tile_id_c = start_pid - NUM_SMS\n\n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)\n num_pid_in_group = GROUP_SIZE_M * num_pid_n\n\n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):\n ^\nTypeError(\"range.__init__() got an unexpected keyword argument 'flatten'\")\n\n", - "limitations": [ - "Real checkpoint weights were paired with synthetic activations, not captured H3 activations.", - "One SM90 GPU, eager WS1 only; no whole-model or TP qualification.", - "The default fixed-tile adapter is slower than the ordinary PyTorch BF16 comparator.", - "The vLLM probe does not qualify canonical autograd or all dW invariance transforms.", - "Strict production resolution remains gated by the test-only descriptor." - ], - "post_measurement_formatting": { - "gpu_rerun_after_formatting": false, - "validation": "Black AST-equivalence check; identical non-import AST and import bindings; local regression rerun", - "files": [ - { - "path": "rl_engine/kernels/ops/h3_ffn_down.py", - "measured_source_sha256": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", - "pr_source_sha256": "4586fd466e8d3c9c13fc9906eca4788ca1d2833d2156f81086a0fe3e42b573b9", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "rl_engine/kernels/ops/h3_down_fp32_backend.py", - "measured_source_sha256": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", - "pr_source_sha256": "4fb231d6548f1ec084dd19edafba76fd4184d6050cc8d182a90c697341ce8258", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "rl_engine/kernels/registry.py", - "measured_source_sha256": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", - "pr_source_sha256": "5e7cb402a3d94c303b5178f3591cfc11c3ad7fc68610262fb1cf4ea0027e7987", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "scripts/validate_h3_ffn_down.py", - "measured_source_sha256": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772", - "pr_source_sha256": "4a16d48437ba330a24ba463bbc35f294d2e2952167472dbc72f2913f9ff75e65", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "scripts/prepare_h3_down_fixture.py", - "measured_source_sha256": "73cfaa8e7fda03bf7ce3fb4e9928d9c01949d843ee4f05407df71e9163bc9778", - "pr_source_sha256": "2accd665d2bfcc49246f9de3ea59482eeac8b0d89314462d11edd1d6582e14b7", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "tests/test_h3_ffn_down.py", - "measured_source_sha256": "bdbb5a86948f1222750508c70fa641b971eeed2dd819de1a222f656b6e2e50b0", - "pr_source_sha256": "eead795bc0dc62a760d3f09b5fbfdf9b0fb16de2bfb415cb983df01d25732308", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - }, - { - "path": "tests/test_h3_down_tooling.py", - "measured_source_sha256": "c6e17f6da1289db2bf26ef83a3a6d63c92b78d95bb9b10082d30a5bcf8f61eca", - "pr_source_sha256": "84a9ca7664edf12c8c3397f3e3383a618e8b38bb60c498512f9fbbce1247d11d", - "changes": "Black 24.4.2 layout and isort 5.13.2 top-level import ordering only", - "non_import_ast_equal": true, - "import_bindings_equal": true - } - ] - }, - "local_pr_validation": { - "status": "passed", - "passed": 85, - "skipped": 3, - "command": "python -m pytest -q tests/test_h3_ffn_down.py tests/test_h3_down_tooling.py tests/test_h3_down_fixture.py tests/test_tolerance_contract.py tests/test_kernel_registry.py tests/test_cross_config_contract.py tests/test_cross_config_runtime.py", - "stdout": "....................sss................................................. [ 81%]\n................ [100%]\n85 passed, 3 skipped in 7.10s\n", - "checks": [ - "Black 24.4.2, line length 100", - "isort 5.13.2, Black profile, line length 100", - "flake8 7.0.0, max line length 100, E203/E704 ignored", - "Ruff", - "bash -n scripts/run_h3_down_h100.sh", - "git diff --check" - ], - "scope": "Local CPU-safe regression on final formatted sources; three full-width SM90 tests skipped. The earlier 64-test H100 run is preserved separately." - }, - "evidence_files": [ - { - "file": "h100-cancellation.json", - "bytes": 244600, - "sha256": "608483ed60de33c388e4bb7bc090fac71619a0d31deceefdac12ea099a6fcb09" - }, - { - "file": "h100-random.json", - "bytes": 244429, - "sha256": "97652ca2c83ba42e821a98fbc2810000152f22af2dffb826b433ce6eabcad1b0" - }, - { - "file": "h100-replays.json", - "bytes": 272824, - "sha256": "e47d4173b52ff3b2d68b314227710e91896c9753b2c485942bb4c585cf2cc89f" - }, - { - "file": "h100-swiglu-like.json", - "bytes": 244560, - "sha256": "a8f615dc2ac313cab0c872da84098e479dab69b2ad74bb5ae95e464433800025" - } - ] - } - } - } -} diff --git a/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json b/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json deleted file mode 100644 index 46ba0a134..000000000 --- a/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json +++ /dev/null @@ -1,10459 +0,0 @@ -{ - "schema_version": 1, - "gpu": "h100", - "provenance": { - "derivative": "Sanitized historical evidence; only execution paths replaced with portable placeholders. All measurements and findings retained. No new GPU run.", - "original_commit": "006021ca9875cdc4ae66af7ed184514070f3274a", - "original_path": "docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json" - }, - "artifacts": { - "h100-swiglu-like.json": { - "original_sha256": "a8f615dc2ac313cab0c872da84098e479dab69b2ad74bb5ae95e464433800025", - "format": "json", - "content": { - "schema": "h3-down-measured-case-part-v1", - "source_report_sha256": "12a76ef4a70492cdbdea75dd60456d4b5da9088425bcfdd50aaa5c6fd5928bf8", - "case_indexes": [ - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31 - ], - "cases": [ - { - "rows": 1, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.006100177764892578, - "max_error_over_tolerance": 0.067171111702919, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0007508695125579834, - "max_error_over_tolerance": 0.0071157063357532024, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.05316777154803276, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.7511414587497711, - "samples_ms": [ - 1.3634003698825836, - 0.8582398295402527, - 0.5834773182868958, - 0.7511414587497711, - 0.5164369940757751 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2127029001712799, - "samples_ms": [ - 0.2271793782711029, - 0.22742152214050293, - 0.21224841475486755, - 0.21011754870414734, - 0.2127029001712799 - ], - "peak_allocated_bytes": 221347840, - "incremental_peak_bytes": 28672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.19888952374458313, - "samples_ms": [ - 0.2205371856689453, - 0.20452961325645447, - 0.19888952374458313, - 0.19580498337745667, - 0.19642338156700134 - ], - "peak_allocated_bytes": 376721408, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.2423345148563385, - "samples_ms": [ - 4.046555608510971, - 4.976723343133926, - 4.653695970773697, - 3.2874606549739838, - 4.2423345148563385 - ], - "peak_allocated_bytes": 531011584, - "incremental_peak_bytes": 309692416, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08771941065788269, - "samples_ms": [ - 0.10984763503074646, - 0.09224936366081238, - 0.08678808808326721, - 0.08628144860267639, - 0.08771941065788269 - ], - "peak_allocated_bytes": 221329920, - "incremental_peak_bytes": 10752, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8339323103427887, - "samples_ms": [ - 0.9573549032211304, - 0.6110593676567078, - 0.8751153945922852, - 0.8339323103427887, - 0.4051104187965393 - ], - "peak_allocated_bytes": 375499264, - "incremental_peak_bytes": 154180096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "c4db5317694fc22eb43d88a37976e33bc2cb1993b098abab6a977c7cfa6d1921", - "cubin_sha256": "8dc3fe06cd084eebd9b382f4b3018f37764b990d0034204daeb838b8c585e17d", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "1981819fe6a10c9802919e16e359a739f2b426d7e27def330c5be4a7e2a60e48", - "cubin_sha256": "d9bd8ed9e45e3b73b861475089a89d7fdc50e4033d0f040a8ec1ea1df3f91a47", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 1, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 5376 - ] - }, - "dx": { - "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43", - "dtype": "torch.bfloat16", - "shape": [ - 1, - 14336 - ] - }, - "dw": { - "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 31, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00782012939453125, - "max_error_over_tolerance": 0.08643997460603714, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009520649909973145, - "max_error_over_tolerance": 0.009015342220664024, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078125, - "max_error_over_tolerance": 0.055741358548402786, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.49997493624687195, - "samples_ms": [ - 0.5282871425151825, - 0.5064867436885834, - 0.49997493624687195, - 0.49972906708717346, - 0.4875659942626953 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.20970404148101807, - "samples_ms": [ - 0.23807212710380554, - 0.2144239842891693, - 0.20970404148101807, - 0.20581483840942383, - 0.20891427993774414 - ], - "peak_allocated_bytes": 224250880, - "incremental_peak_bytes": 888832, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.20439177751541138, - "samples_ms": [ - 0.2192482352256775, - 0.20439177751541138, - 0.21205097436904907, - 0.19450858235359192, - 0.1872330904006958 - ], - "peak_allocated_bytes": 378764288, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.554342478513718, - "samples_ms": [ - 4.593264311552048, - 4.323776811361313, - 4.380952566862106, - 4.554342478513718, - 4.684560000896454 - ], - "peak_allocated_bytes": 537462784, - "incremental_peak_bytes": 314100736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.09334459900856018, - "samples_ms": [ - 0.10614469647407532, - 0.096101313829422, - 0.09334459900856018, - 0.09072944521903992, - 0.09304285049438477 - ], - "peak_allocated_bytes": 223695360, - "incremental_peak_bytes": 333312, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.1095404624938965, - "samples_ms": [ - 1.302909106016159, - 1.3855025172233582, - 1.1095404624938965, - 0.4470720887184143, - 0.7269084453582764 - ], - "peak_allocated_bytes": 378724864, - "incremental_peak_bytes": 155362816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 31, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6a603f6d21ea02cd5963cf81bb7d1446bd96af21989e0ef649a314e6f195ef24", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 5376 - ] - }, - "dx": { - "sha256": "3caf4cfab06db14f91702f774666b715c08cfc28c44b0d1a27ec1fb896b39df2", - "dtype": "torch.bfloat16", - "shape": [ - 31, - 14336 - ] - }, - "dw": { - "sha256": "a9e413a88de121e78c0b957b14a4fffbd905590c5646e1905db3ee2e4588f611", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00782012939453125, - "max_error_over_tolerance": 0.08643997460603714, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009520649909973145, - "max_error_over_tolerance": 0.009015342220664024, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007812023162841797, - "max_error_over_tolerance": 0.055688682943582535, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.472191721200943, - "samples_ms": [ - 0.5241893231868744, - 0.4849061369895935, - 0.472191721200943, - 0.4712902009487152, - 0.46755746006965637 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.19920989871025085, - "samples_ms": [ - 0.2119503915309906, - 0.20406395196914673, - 0.19920989871025085, - 0.1966916024684906, - 0.19863247871398926 - ], - "peak_allocated_bytes": 224347648, - "incremental_peak_bytes": 917504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.14410540461540222, - "samples_ms": [ - 0.1502782106399536, - 0.14410540461540222, - 0.15384703874588013, - 0.13763830065727234, - 0.13315677642822266 - ], - "peak_allocated_bytes": 377570816, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 2.8609633445739746, - "samples_ms": [ - 2.8609633445739746, - 3.24355810880661, - 2.8083324432373047, - 2.8740428388118744, - 2.657875418663025 - ], - "peak_allocated_bytes": 536416256, - "incremental_peak_bytes": 312986112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08560344576835632, - "samples_ms": [ - 0.09668990969657898, - 0.08917227387428284, - 0.08560344576835632, - 0.08519366383552551, - 0.08498504757881165 - ], - "peak_allocated_bytes": 223774208, - "incremental_peak_bytes": 344064, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8179731667041779, - "samples_ms": [ - 0.8179731667041779, - 0.9407177567481995, - 0.8269548416137695, - 0.36824867129325867, - 0.30737370252609253 - ], - "peak_allocated_bytes": 378832384, - "incremental_peak_bytes": 155402240, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 32, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "c6c699d786504292156fc35e27dcdaab91185830934b117913bfa8e4aff20516", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 5376 - ] - }, - "dx": { - "sha256": "56f6ad0df9ec971093c734e6467f691f20a9e46408f53616fb00badb1ae2a60b", - "dtype": "torch.bfloat16", - "shape": [ - 32, - 14336 - ] - }, - "dw": { - "sha256": "a980324f5122698c24c2e24cb8cc855c9f698ad7cb657c9e19e06077ab272bf5", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 33, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007819890975952148, - "max_error_over_tolerance": 0.08643733710050583, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009521245956420898, - "max_error_over_tolerance": 0.009015905670821667, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.014435768127441406, - "max_error_over_tolerance": 0.07634487003087997, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5420371890068054, - "samples_ms": [ - 0.508435070514679, - 0.6049051880836487, - 0.5784481763839722, - 0.4877038300037384, - 0.5420371890068054 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.1995675265789032, - "samples_ms": [ - 0.20760297775268555, - 0.1995675265789032, - 0.19385293126106262, - 0.2023652195930481, - 0.1921132206916809 - ], - "peak_allocated_bytes": 224444928, - "incremental_peak_bytes": 946176, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.19453838467597961, - "samples_ms": [ - 0.21283701062202454, - 0.19798427820205688, - 0.1898147165775299, - 0.19311904907226562, - 0.19453838467597961 - ], - "peak_allocated_bytes": 380162560, - "incremental_peak_bytes": 156663808, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.123970866203308, - "samples_ms": [ - 4.327472299337387, - 4.013523459434509, - 4.435192793607712, - 3.1566955149173737, - 4.123970866203308 - ], - "peak_allocated_bytes": 539156480, - "incremental_peak_bytes": 315657728, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.08546561002731323, - "samples_ms": [ - 0.09231641888618469, - 0.08748844265937805, - 0.08546561002731323, - 0.08390471339225769, - 0.08344277739524841 - ], - "peak_allocated_bytes": 223853568, - "incremental_peak_bytes": 354816, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.2046098709106445, - "samples_ms": [ - 1.2046098709106445, - 1.2146830558776855, - 1.2108273804187775, - 1.1087283492088318, - 0.4986859858036041 - ], - "peak_allocated_bytes": 378940416, - "incremental_peak_bytes": 155441664, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 1, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 1, - 224 - ] - }, - "canonical_active_rows": 33, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 5376 - ] - }, - "dx": { - "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60", - "dtype": "torch.bfloat16", - "shape": [ - 33, - 14336 - ] - }, - "dw": { - "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 127, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00780034065246582, - "max_error_over_tolerance": 0.08598984032869339, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009499192237854004, - "max_error_over_tolerance": 0.009035039693117142, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015492439270019531, - "max_error_over_tolerance": 0.08494006842374802, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.4916340112686157, - "samples_ms": [ - 0.5290843546390533, - 0.5001910030841827, - 0.4916340112686157, - 0.48222020268440247, - 0.47730281949043274 - ], - "peak_allocated_bytes": 231722496, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2099648118019104, - "samples_ms": [ - 0.21921470761299133, - 0.2099648118019104, - 0.215776264667511, - 0.205211341381073, - 0.20548328757286072 - ], - "peak_allocated_bytes": 233998336, - "incremental_peak_bytes": 3641344, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.23742765188217163, - "samples_ms": [ - 0.25482848286628723, - 0.24598091840744019, - 0.23742765188217163, - 0.23298710584640503, - 0.2289116382598877 - ], - "peak_allocated_bytes": 389959680, - "incremental_peak_bytes": 159602688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.725195467472076, - "samples_ms": [ - 5.029212683439255, - 4.744764417409897, - 4.252210259437561, - 4.3155737221241, - 4.725195467472076 - ], - "peak_allocated_bytes": 563719680, - "incremental_peak_bytes": 333362688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.10073184967041016, - "samples_ms": [ - 0.10984763503074646, - 0.10212883353233337, - 0.10073184967041016, - 0.09812414646148682, - 0.09807571768760681 - ], - "peak_allocated_bytes": 231722496, - "incremental_peak_bytes": 1365504, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.1009685695171356, - "samples_ms": [ - 1.3191774487495422, - 0.9455867111682892, - 1.1009685695171356, - 0.9224340319633484, - 1.1019520461559296 - ], - "peak_allocated_bytes": 389504512, - "incremental_peak_bytes": 159147520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 127, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "69de6ffba99e13cb0f50b6019f6ab4a60222501650d11de89b0c0d0844a0c7ad", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 5376 - ] - }, - "dx": { - "sha256": "697a3795ae80f6e5b4b014270998def2b4763acb973ab8134213cb80f19e8fb8", - "dtype": "torch.bfloat16", - "shape": [ - 127, - 14336 - ] - }, - "dw": { - "sha256": "f4999860992d23b6ab509cdb088ac0c52df6e23763b7ab78446b19cc94fb2789", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 128, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00780034065246582, - "max_error_over_tolerance": 0.08598984032869339, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009499192237854004, - "max_error_over_tolerance": 0.009035039693117142, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015538215637207031, - "max_error_over_tolerance": 0.0854954943060875, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6464831531047821, - "samples_ms": [ - 0.705275684595108, - 0.6767474114894867, - 0.6464831531047821, - 0.6321556866168976, - 0.6436444818973541 - ], - "peak_allocated_bytes": 231804928, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.27082115411758423, - "samples_ms": [ - 0.30130520462989807, - 0.2724193036556244, - 0.26627257466316223, - 0.26065483689308167, - 0.27082115411758423 - ], - "peak_allocated_bytes": 234098688, - "incremental_peak_bytes": 3670016, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.24049729108810425, - "samples_ms": [ - 0.2527572214603424, - 0.24078786373138428, - 0.23163482546806335, - 0.23698434233665466, - 0.24049729108810425 - ], - "peak_allocated_bytes": 384569344, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.078967660665512, - "samples_ms": [ - 4.93263453245163, - 5.035568028688431, - 5.260847508907318, - 5.078967660665512, - 5.102559924125671 - ], - "peak_allocated_bytes": 558902784, - "incremental_peak_bytes": 328474112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.11829286813735962, - "samples_ms": [ - 0.13687461614608765, - 0.12293830513954163, - 0.11829286813735962, - 0.11602789163589478, - 0.11521577835083008 - ], - "peak_allocated_bytes": 231804928, - "incremental_peak_bytes": 1376256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.49502232670784, - "samples_ms": [ - 1.6067475080490112, - 1.5815980732440948, - 1.4262758195400238, - 1.0990574955940247, - 1.49502232670784 - ], - "peak_allocated_bytes": 389615616, - "incremental_peak_bytes": 159186944, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 2, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 2, - 224 - ] - }, - "canonical_active_rows": 128, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "ba7eaec4960966c7b2a37c6e44c020e67304a0f7877a159fd000c8878500dd90", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 5376 - ] - }, - "dx": { - "sha256": "1dc23d75e50aef565ad84ae3dc21d07806176a689e5b906ee775724a71f65a9d", - "dtype": "torch.bfloat16", - "shape": [ - 128, - 14336 - ] - }, - "dw": { - "sha256": "f21ca96fd9b83e2a051392a250ce2f39862555f517c3e961445047c5873ee24f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 129, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0078012943267822266, - "max_error_over_tolerance": 0.08598456531763077, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009499192237854004, - "max_error_over_tolerance": 0.009035039693117142, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015491485595703125, - "max_error_over_tolerance": 0.08537446707487106, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.6536431610584259, - "samples_ms": [ - 0.8461922407150269, - 0.6669200956821442, - 0.6536431610584259, - 0.6260201334953308, - 0.6349869072437286 - ], - "peak_allocated_bytes": 231888896, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.2928338944911957, - "samples_ms": [ - 0.31632184982299805, - 0.2928338944911957, - 0.29468536376953125, - 0.2854093909263611, - 0.2806149423122406 - ], - "peak_allocated_bytes": 234200576, - "incremental_peak_bytes": 3698688, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.3973469138145447, - "samples_ms": [ - 0.40783360600471497, - 0.38995593786239624, - 0.3815554082393646, - 0.3973469138145447, - 0.48520416021347046 - ], - "peak_allocated_bytes": 391910912, - "incremental_peak_bytes": 161409024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 5.093999207019806, - "samples_ms": [ - 5.492702126502991, - 5.749542266130447, - 4.561234265565872, - 5.093999207019806, - 5.001761019229889 - ], - "peak_allocated_bytes": 565443584, - "incremental_peak_bytes": 334941696, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.15628337860107422, - "samples_ms": [ - 0.18086284399032593, - 0.1616552472114563, - 0.15628337860107422, - 0.1537799835205078, - 0.15249475836753845 - ], - "peak_allocated_bytes": 231888896, - "incremental_peak_bytes": 1387008, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.5014559030532837, - "samples_ms": [ - 1.3852529227733612, - 1.226574182510376, - 1.5177838504314423, - 1.5014559030532837, - 1.6144178807735443 - ], - "peak_allocated_bytes": 389728256, - "incremental_peak_bytes": 159226368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 3, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 3, - 224 - ] - }, - "canonical_active_rows": 129, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 5376 - ] - }, - "dx": { - "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8", - "dtype": "torch.bfloat16", - "shape": [ - 129, - 14336 - ] - }, - "dw": { - "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 256, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007821321487426758, - "max_error_over_tolerance": 0.08593791723251343, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009765326976776123, - "max_error_over_tolerance": 0.009236739948391914, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.015624523162841797, - "max_error_over_tolerance": 0.08663392066955566, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5091354250907898, - "samples_ms": [ - 0.594262033700943, - 0.5234144628047943, - 0.5091354250907898, - 0.5043074488639832, - 0.5023069679737091 - ], - "peak_allocated_bytes": 242360320, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.23020803928375244, - "samples_ms": [ - 0.24154409766197205, - 0.24346262216567993, - 0.23020803928375244, - 0.22159144282341003, - 0.22471323609352112 - ], - "peak_allocated_bytes": 246947840, - "incremental_peak_bytes": 7340032, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.25147944688796997, - "samples_ms": [ - 0.2661608159542084, - 0.25453418493270874, - 0.2458915114402771, - 0.25030598044395447, - 0.25147944688796997 - ], - "peak_allocated_bytes": 393748480, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 3.2671354711055756, - "samples_ms": [ - 2.359338104724884, - 2.3001059889793396, - 3.2671354711055756, - 3.9541684091091156, - 3.9238929748535156 - ], - "peak_allocated_bytes": 586439168, - "incremental_peak_bytes": 346831360, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1126602292060852, - "samples_ms": [ - 0.12792646884918213, - 0.11543557047843933, - 0.1126602292060852, - 0.11115893721580505, - 0.11189281940460205 - ], - "peak_allocated_bytes": 242360320, - "incremental_peak_bytes": 2752512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.8542537689208984, - "samples_ms": [ - 0.9875744581222534, - 0.8467361330986023, - 0.9040683507919312, - 0.8542537689208984, - 0.4503205418586731 - ], - "peak_allocated_bytes": 403841024, - "incremental_peak_bytes": 164233216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 4, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 4, - 224 - ] - }, - "canonical_active_rows": 256, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "948b289d0ffaf1e67d6aad8e9ae565c21a57ed5809a56531cd802a673a7a1186", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 5376 - ] - }, - "dx": { - "sha256": "d5f3c01d93a8a90120e0c09a1f7706435e4a615acc1dff5b5e32bd14bdc6d790", - "dtype": "torch.bfloat16", - "shape": [ - 256, - 14336 - ] - }, - "dw": { - "sha256": "71cd2c2acd3131879911a8cd2c79299f560f0a160c28e2d79a0a67272b8fd152", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 513, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00783395767211914, - "max_error_over_tolerance": 0.08669471740722656, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009765625, - "max_error_over_tolerance": 0.009295407682657242, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.03013134002685547, - "max_error_over_tolerance": 0.11083704233169556, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 0.5357936024665833, - "samples_ms": [ - 0.5600899457931519, - 0.52647665143013, - 0.5357936024665833, - 0.5212984979152679, - 0.7661432027816772 - ], - "peak_allocated_bytes": 262334464, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.5339905619621277, - "samples_ms": [ - 0.6604790687561035, - 0.5883835256099701, - 0.5339905619621277, - 0.375404953956604, - 0.3919526934623718 - ], - "peak_allocated_bytes": 271527424, - "incremental_peak_bytes": 14708736, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.4009269177913666, - "samples_ms": [ - 0.6171725690364838, - 0.4009269177913666, - 0.39183348417282104, - 0.39962679147720337, - 0.4404895007610321 - ], - "peak_allocated_bytes": 432406016, - "incremental_peak_bytes": 175587328, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 2.8889253735542297, - "samples_ms": [ - 2.820633351802826, - 2.8889253735542297, - 2.8440728783607483, - 2.9198452830314636, - 3.1279921531677246 - ], - "peak_allocated_bytes": 661960704, - "incremental_peak_bytes": 405142016, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.1828409731388092, - "samples_ms": [ - 0.19228458404541016, - 0.1829490065574646, - 0.18090009689331055, - 0.17902255058288574, - 0.1828409731388092 - ], - "peak_allocated_bytes": 262334464, - "incremental_peak_bytes": 5515776, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 0.7188506424427032, - "samples_ms": [ - 0.7435493171215057, - 0.7229670882225037, - 0.7097125053405762, - 0.7188506424427032, - 0.6750933825969696 - ], - "peak_allocated_bytes": 431183872, - "incremental_peak_bytes": 174365184, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 9, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 9, - 224 - ] - }, - "canonical_active_rows": 513, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "305525b6b8360ad1215647dc5b4412b9cf085864533f03ffc40ecb9cb77555ea", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 5376 - ] - }, - "dx": { - "sha256": "8bb06fd1bbe46d5a333c3e50e1906131da3f6e9a8049479f7595a20ccb79a1cf", - "dtype": "torch.bfloat16", - "shape": [ - 513, - 14336 - ] - }, - "dw": { - "sha256": "54314948f05a018ef73b260df92cfb432d1fca3506a865f9ff33f9243878b401", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 1024, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007846832275390625, - "max_error_over_tolerance": 0.08643604815006256, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009739995002746582, - "max_error_over_tolerance": 0.009267088025808334, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.031244277954101562, - "max_error_over_tolerance": 0.11963395029306412, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.003596931695938, - "samples_ms": [ - 1.0749511420726776, - 1.0326988995075226, - 0.9927935898303986, - 0.9920410811901093, - 1.003596931695938 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 0.6674230098724365, - "samples_ms": [ - 0.6864443421363831, - 0.6674230098724365, - 0.6535425782203674, - 0.6415434181690216, - 0.6711967289447784 - ], - "peak_allocated_bytes": 320372736, - "incremental_peak_bytes": 29360128, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 0.5908161401748657, - "samples_ms": [ - 0.6029978394508362, - 0.5908161401748657, - 0.593278557062149, - 0.5755200982093811, - 0.5740784108638763 - ], - "peak_allocated_bytes": 445153280, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 4.697073251008987, - "samples_ms": [ - 4.6738311648368835, - 4.697073251008987, - 4.740521311759949, - 4.760138690471649, - 4.647858440876007 - ], - "peak_allocated_bytes": 749822976, - "incremental_peak_bytes": 458810368, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.27285143733024597, - "samples_ms": [ - 0.2916380763053894, - 0.2776235342025757, - 0.27285143733024597, - 0.26857107877731323, - 0.2683289349079132 - ], - "peak_allocated_bytes": 302022656, - "incremental_peak_bytes": 11010048, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.4369338750839233, - "samples_ms": [ - 1.4369338750839233, - 1.3857372105121613, - 1.5310943126678467, - 1.4232508838176727, - 1.5554651618003845 - ], - "peak_allocated_bytes": 485523456, - "incremental_peak_bytes": 194510848, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 16, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 16, - 224 - ] - }, - "canonical_active_rows": 1024, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "5a8c854fa60235b480d333b01b7017b1319c0daf54dff2338dc625018fecaf70", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 5376 - ] - }, - "dx": { - "sha256": "208e5216e4a16a5accf3a50ad3ef4fba4ff062322f0fb8dd1b4b364389095238", - "dtype": "torch.bfloat16", - "shape": [ - 1024, - 14336 - ] - }, - "dw": { - "sha256": "970175a5ac5a8e206233200b0865d3df694adf5888d5b7b441ce14b0859f270b", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 2048, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007836103439331055, - "max_error_over_tolerance": 0.08683990687131882, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009764134883880615, - "max_error_over_tolerance": 0.00927541870623827, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.031280517578125, - "max_error_over_tolerance": 0.11995884031057358, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 1.4924593269824982, - "samples_ms": [ - 1.575138419866562, - 1.537363976240158, - 1.4924593269824982, - 1.4637857675552368, - 1.4722123742103577 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 1.1428110301494598, - "samples_ms": [ - 1.1428110301494598, - 1.1433251202106476, - 1.1936947703361511, - 1.1115595698356628, - 1.0797232389450073 - ], - "peak_allocated_bytes": 419495936, - "incremental_peak_bytes": 58720256, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 1.0681450366973877, - "samples_ms": [ - 1.0858029127120972, - 1.0655000805854797, - 1.0727979242801666, - 1.0573789477348328, - 1.0681450366973877 - ], - "peak_allocated_bytes": 514916352, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 6.326321512460709, - "samples_ms": [ - 5.975883454084396, - 6.353158503770828, - 6.0294009745121, - 6.326321512460709, - 13.371232897043228 - ], - "peak_allocated_bytes": 970115072, - "incremental_peak_bytes": 609339392, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.5012601613998413, - "samples_ms": [ - 0.49800798296928406, - 0.5012601613998413, - 0.5335211753845215, - 0.5161166191101074, - 0.49573928117752075 - ], - "peak_allocated_bytes": 382795776, - "incremental_peak_bytes": 22020096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 1.7548240721225739, - "samples_ms": [ - 1.710943877696991, - 1.753535121679306, - 1.828797161579132, - 1.8369108438491821, - 1.7548240721225739 - ], - "peak_allocated_bytes": 595656704, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 32, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 32, - 224 - ] - }, - "canonical_active_rows": 2048, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "4fc22b3ef0763c90709fdcf83d304de53076d9e5010a263e1b34d9d6cfe59be4", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 5376 - ] - }, - "dx": { - "sha256": "a0ef242ffebeb3b2b716559c6c25462059938b7b56e791c2d21a80be9e4f9005", - "dtype": "torch.bfloat16", - "shape": [ - 2048, - 14336 - ] - }, - "dw": { - "sha256": "da2b0000f22f1145ff004e49eeab6c826babe129a04f3890576321d747b58075", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 4096, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007841348648071289, - "max_error_over_tolerance": 0.08670263737440109, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009782612323760986, - "max_error_over_tolerance": 0.009300120174884796, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06252670288085938, - "max_error_over_tolerance": 0.14839504659175873, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 2.8487443923950195, - "samples_ms": [ - 2.87516787648201, - 2.8021037578582764, - 2.834375947713852, - 2.8487443923950195, - 2.903975546360016 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 2.128344029188156, - "samples_ms": [ - 2.1580159664154053, - 2.1231621503829956, - 2.089381217956543, - 2.128344029188156, - 2.1305643022060394 - ], - "peak_allocated_bytes": 617742336, - "incremental_peak_bytes": 117440512, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 2.0004957914352417, - "samples_ms": [ - 2.03578919172287, - 2.0004957914352417, - 2.0807236433029175, - 1.9929297268390656, - 1.990877091884613 - ], - "peak_allocated_bytes": 654442496, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 9.617339819669724, - "samples_ms": [ - 9.428001940250397, - 9.563304483890533, - 9.74905863404274, - 10.471254587173462, - 9.617339819669724 - ], - "peak_allocated_bytes": 1410699264, - "incremental_peak_bytes": 910397440, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 0.9003691375255585, - "samples_ms": [ - 1.0827071964740753, - 1.191694289445877, - 0.9003691375255585, - 0.8887462317943573, - 0.8913837373256683 - ], - "peak_allocated_bytes": 544342016, - "incremental_peak_bytes": 44040192, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 2.7191415429115295, - "samples_ms": [ - 2.706039696931839, - 2.7057379484176636, - 2.7206838130950928, - 2.719506621360779, - 2.7191415429115295 - ], - "peak_allocated_bytes": 815923200, - "incremental_peak_bytes": 315621376, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 64, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 64, - 224 - ] - }, - "canonical_active_rows": 4096, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "a60e34372727dd9149217014270081afc07bd4915d95e4a7a85a237c08a4b178", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 5376 - ] - }, - "dx": { - "sha256": "a74718d8c858efb3bac5fc50ba60010b8882e12a46510a93dc2ede260e01296c", - "dtype": "torch.bfloat16", - "shape": [ - 4096, - 14336 - ] - }, - "dw": { - "sha256": "a96f78202954c57a84b1acc245f5d8131a12a9f7d9ffef98bc41847715d4167c", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 8192, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007861852645874023, - "max_error_over_tolerance": 0.08676071465015411, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009774267673492432, - "max_error_over_tolerance": 0.0092932665720582, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.06272697448730469, - "max_error_over_tolerance": 0.14868396520614624, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 6.731804460287094, - "samples_ms": [ - 6.731804460287094, - 6.341461092233658, - 6.911188364028931, - 6.427716463804245, - 6.800435483455658 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 4.341848194599152, - "samples_ms": [ - 4.340160638093948, - 4.464488476514816, - 4.254065454006195, - 4.358276724815369, - 4.341848194599152 - ], - "peak_allocated_bytes": 1014235136, - "incremental_peak_bytes": 234881024, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 4.210811108350754, - "samples_ms": [ - 4.122566431760788, - 4.205178469419479, - 4.225626587867737, - 4.210811108350754, - 4.668742418289185 - ], - "peak_allocated_bytes": 933494784, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 19.256968051195145, - "samples_ms": [ - 19.256968051195145, - 18.107809126377106, - 19.30183172225952, - 19.30096000432968, - 19.102156162261963 - ], - "peak_allocated_bytes": 2291867648, - "incremental_peak_bytes": 1512513536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 1.716524362564087, - "samples_ms": [ - 1.8020570278167725, - 1.7209723591804504, - 1.716524362564087, - 1.7112195491790771, - 1.7103217542171478 - ], - "peak_allocated_bytes": 867434496, - "incremental_peak_bytes": 88080384, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 5.995567888021469, - "samples_ms": [ - 5.205079913139343, - 5.2115097641944885, - 5.995567888021469, - 6.085146218538284, - 8.324682712554932 - ], - "peak_allocated_bytes": 1256456192, - "incremental_peak_bytes": 477102080, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 128, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 128, - 224 - ] - }, - "canonical_active_rows": 8192, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "65c1cf45a27c17a87b82850851f40e8e8e91d13925ee21882e513b8dcb1d4d0c", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 5376 - ] - }, - "dx": { - "sha256": "5c42594fdd1a798ef93cd0bfce380ed91ed081746c84b269bff4008eb26b580e", - "dtype": "torch.bfloat16", - "shape": [ - 8192, - 14336 - ] - }, - "dw": { - "sha256": "073de23aa27b3c4b2b8f3a33d7b249d2ad529fe1883ac3676370a88ac7601362", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 16385, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007851600646972656, - "max_error_over_tolerance": 0.08705637603998184, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009774565696716309, - "max_error_over_tolerance": 0.009300513193011284, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12546157836914062, - "max_error_over_tolerance": 0.16740718483924866, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 20.043708384037018, - "samples_ms": [ - 20.18143981695175, - 20.07604017853737, - 19.939277321100235, - 19.863557070493698, - 20.043708384037018 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 8.881796151399612, - "samples_ms": [ - 9.00673121213913, - 8.875839412212372, - 8.893750607967377, - 8.881796151399612, - 8.768562227487564 - ], - "peak_allocated_bytes": 1807319040, - "incremental_peak_bytes": 469790720, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 9.493160992860794, - "samples_ms": [ - 9.27766039967537, - 9.493160992860794, - 9.644988924264908, - 9.2148557305336, - 9.505551308393478 - ], - "peak_allocated_bytes": 2138853376, - "incremental_peak_bytes": 801325056, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 43.05011406540871, - "samples_ms": [ - 43.04959625005722, - 43.427929282188416, - 43.05969923734665, - 42.90411248803139, - 43.05011406540871 - ], - "peak_allocated_bytes": 4701607936, - "incremental_peak_bytes": 3364079616, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 3.1884387135505676, - "samples_ms": [ - 3.1987205147743225, - 3.1884387135505676, - 3.173545002937317, - 3.177415579557419, - 3.2145678997039795 - ], - "peak_allocated_bytes": 1513699840, - "incremental_peak_bytes": 176171520, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 11.09558716416359, - "samples_ms": [ - 10.03454253077507, - 10.854113847017288, - 11.104598641395569, - 11.09558716416359, - 11.109769344329834 - ], - "peak_allocated_bytes": 2137631232, - "incremental_peak_bytes": 800102912, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 257, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 257, - 224 - ] - }, - "canonical_active_rows": 16385, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "801986e1776078be2cbc69588869fc45075cd3c8a1110e8ef659d6af8c6a6e66", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 5376 - ] - }, - "dx": { - "sha256": "aa68ce734273a068db9d32d95009990e8845a2287544e6bb806309bed9c68f5b", - "dtype": "torch.bfloat16", - "shape": [ - 16385, - 14336 - ] - }, - "dw": { - "sha256": "b1f7ef34a54388d33cba9e61ece11f1e395f32394226c44ea8eae2c8121554ff", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 24577, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.00786590576171875, - "max_error_over_tolerance": 0.08697982132434845, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.000977933406829834, - "max_error_over_tolerance": 0.009302553720772266, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12613296508789062, - "max_error_over_tolerance": 0.16935373842716217, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "inactive_padding" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "inactive_padding" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 31.051654368638992, - "samples_ms": [ - 30.949130654335022, - 31.17082267999649, - 31.01911023259163, - 31.051654368638992, - 32.34020248055458 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 18.86589080095291, - "samples_ms": [ - 19.044052809476852, - 18.86589080095291, - 17.993338406085968, - 18.721047788858414, - 21.820947527885437 - ], - "peak_allocated_bytes": 2600304640, - "incremental_peak_bytes": 704671744, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 14.459915459156036, - "samples_ms": [ - 14.484800398349762, - 14.459915459156036, - 14.379594475030899, - 14.104023575782776, - 14.869511127471924 - ], - "peak_allocated_bytes": 3019919360, - "incremental_peak_bytes": 1124286464, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 70.6416592001915, - "samples_ms": [ - 70.6416592001915, - 71.31434231996536, - 71.67328521609306, - 69.40366700291634, - 70.14808431267738 - ], - "peak_allocated_bytes": 6786906112, - "incremental_peak_bytes": 4891273216, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 5.352184176445007, - "samples_ms": [ - 5.024075508117676, - 5.151830613613129, - 5.389474332332611, - 5.390428006649017, - 5.352184176445007 - ], - "peak_allocated_bytes": 2159884800, - "incremental_peak_bytes": 264251904, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 18.56180652976036, - "samples_ms": [ - 17.890069633722305, - 18.56180652976036, - 19.248690456151962, - 18.597692251205444, - 18.478691577911377 - ], - "peak_allocated_bytes": 3018697216, - "incremental_peak_bytes": 1123064320, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "049d3c9b60963d093914fb6be2c91c69d93d6dfc1fd6cc3ebad2059a530569e5", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 385, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "5d2af1877b1657c36965e4c35fc046f7be8d69dfa3db3e0193b0423d6bb351f6", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 385, - 224 - ] - }, - "canonical_active_rows": 24577, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "9095fd83587eea976861650c3dc4edabd12321eb89a8f8dea2f530b499f1b058", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 5376 - ] - }, - "dx": { - "sha256": "97f01ad55ca2b6158cde2dc3e8756a7d46bd97ea16b0611bd97b7e22355ffcba", - "dtype": "torch.bfloat16", - "shape": [ - 24577, - 14336 - ] - }, - "dw": { - "sha256": "3cee55c58793b1a9920e6bbdd266cfe9d8ab6ae1cd66f439129f5b214309302f", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - }, - { - "rows": 32768, - "input_family": "swiglu_like", - "checks": [ - { - "judgment": "forward_accuracy", - "tolerance": { - "judgment": "forward_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.05, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.007860183715820312, - "max_error_over_tolerance": 0.08708541095256805, - "failing_elements": 0, - "tensor": "y", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.0009776651859283447, - "max_error_over_tolerance": 0.009305901825428009, - "failing_elements": 0, - "tensor": "dx", - "transform": "fp32_reference" - }, - { - "judgment": "gradient_accuracy", - "tolerance": { - "judgment": "gradient_accuracy", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "tolerance", - "atol": 0.1, - "rtol": 0.02, - "comparison_lhs_role": "bf16_candidate", - "comparison_rhs_role": "fp32_reference", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "max_abs_error": 0.12707138061523438, - "max_error_over_tolerance": 0.1699322909116745, - "failing_elements": 0, - "tensor": "dw", - "transform": "fp32_reference" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "repeat" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "repeat" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "permuted_logical_rows" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "permuted_logical_rows" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "chunk31_single_backward" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "chunk31_single_backward" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "strided_input_and_weight" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "strided_input_and_weight" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "batch2_reshape" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dw", - "transform": "batch2_reshape" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "singleton_vs_batch" - }, - { - "judgment": "forward_invariance", - "tolerance": { - "judgment": "forward_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "y", - "transform": "other_rows_mutated" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "singleton_vs_batch" - }, - { - "judgment": "gradient_invariance", - "tolerance": { - "judgment": "gradient_invariance", - "op_class": "reduction", - "dtype_name": "bfloat16", - "status": "applicable", - "mode": "bitwise", - "atol": 0.0, - "rtol": 0.0, - "comparison_lhs_role": "transformed_config", - "comparison_rhs_role": "canonical_config", - "backend_profile": "cuda_bf16", - "arch_key": "sm90" - }, - "passed": true, - "mismatched_bytes": 0, - "tensor": "dx", - "transform": "other_rows_mutated" - } - ], - "benchmarks": { - "candidate_forward": { - "median_ms": 41.59888997673988, - "samples_ms": [ - 41.66906327009201, - 41.59888997673988, - 41.56194254755974, - 41.601087898015976, - 41.56932234764099 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dx": { - "median_ms": 37.15309500694275, - "samples_ms": [ - 36.36578842997551, - 37.498798221349716, - 37.37569972872734, - 37.15309500694275, - 36.31233796477318 - ], - "peak_allocated_bytes": 3393191936, - "incremental_peak_bytes": 939524096, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_dw_canonical_order": { - "median_ms": 18.57171207666397, - "samples_ms": [ - 20.10498195886612, - 19.960522651672363, - 18.57171207666397, - 18.499113619327545, - 18.54676753282547 - ], - "peak_allocated_bytes": 2607808512, - "incremental_peak_bytes": 154140672, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "candidate_forward_backward": { - "median_ms": 108.27267915010452, - "samples_ms": [ - 108.27267915010452, - 109.35502871870995, - 108.98803547024727, - 105.77517747879028, - 106.29720613360405 - ], - "peak_allocated_bytes": 7578877952, - "incremental_peak_bytes": 5125210112, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward": { - "median_ms": 7.173832505941391, - "samples_ms": [ - 6.3136667013168335, - 6.678733974695206, - 7.226929068565369, - 7.173832505941391, - 7.515721023082733 - ], - "peak_allocated_bytes": 2805989376, - "incremental_peak_bytes": 352321536, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - }, - "torch_bf16_forward_backward": { - "median_ms": 21.348431706428528, - "samples_ms": [ - 22.704727947711945, - 21.976932883262634, - 20.644009113311768, - 21.064143627882004, - 21.348431706428528 - ], - "peak_allocated_bytes": 3899654144, - "incremental_peak_bytes": 1445986304, - "timing_scope": "synchronized wall time, includes wrapper/copies/padding" - } - }, - "execution_traces": [ - { - "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "fallback": false, - "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", - "tile": [ - 64, - 64, - 32 - ], - "num_warps": 4, - "num_stages": 2, - "fp_fusion": false, - "split_k": false, - "split_policy": "none", - "accumulator_dtype": "float32", - "schedule_id": "fixed-64x64x32-w4-s2-fusion0", - "topology": { - "world_size": 1, - "tensor_parallel_size": 1 - }, - "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", - "weight_layout": "[5376,14336]", - "dw_order": "lexicographic active logical keys; trailing zero pad to 32", - "forward_executed": true, - "input_gradient_executed": true, - "weight_gradient_executed": true, - "qualification": "pending_h3_gpu_evidence", - "forward_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "527daa8b51f7ea9eaa61a467859a9f328eaf5680f2ebbcd7f9ce3dc6d1756ea0", - "cubin_sha256": "c13ae7b0de0550cfce34cdc145fd3870397dc7415d9fbf15d9bc834eaa4dc5bb", - "compiled_num_warps": 4, - "grid": [ - 512, - 84 - ] - }, - "input_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "408c4569eccee245d120e1301ca835d34d6aed427c6515c7f6cdeb9adbfb3455", - "cubin_sha256": "ccbec883a81b858d67f6b6f7da64b6803f2058fb62b16f8f5929ad8b09ac2dcd", - "compiled_num_warps": 4, - "grid": [ - 512, - 224 - ] - }, - "canonical_active_rows": 32768, - "weight_gradient_launch": { - "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", - "triton_version": "3.2.0", - "compiled_kernel_hash": "7e6df3621ec6fafddeb11d35fc08fc97b13b6ba8001295593b9c81c9235ed954", - "cubin_sha256": "4e3983acd354ae954d7a3e6c62471f7cfb70f4f0ce0a18bfa4e666973ed7509a", - "compiled_num_warps": 4, - "grid": [ - 84, - 224 - ] - } - } - ], - "output_digests": { - "y": { - "sha256": "46b761fcf0a9ab3186d792ed8a5fe420a22a76fd2fe30709fb5a446cad834ea8", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 5376 - ] - }, - "dx": { - "sha256": "8b9866742cfe3ed59b577b6a0f27628052499205e7322bb48a66c70afcafae8d", - "dtype": "torch.bfloat16", - "shape": [ - 32768, - 14336 - ] - }, - "dw": { - "sha256": "5ff54c5913b0c7e414a95f29aefcba834e10ed35d837aa01c9b659ec5a4c745d", - "dtype": "torch.bfloat16", - "shape": [ - 5376, - 14336 - ] - } - }, - "passed": true - } - ] - } - } - } -} diff --git a/docs/operators/h3-ffn-down-results.json b/docs/operators/h3-ffn-down-results.json new file mode 100644 index 000000000..9595db62a --- /dev/null +++ b/docs/operators/h3-ffn-down-results.json @@ -0,0 +1,968 @@ +{ + "schema_version": 1, + "gpu": "NVIDIA H100 80GB HBM3", + "provenance": { + "kind": "Selected and aggregated historical evidence, not a new execution or lossless export", + "source_commit": "d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6", + "original_bundles_sha256": { + "h100-summary.json": "3a44e25a9397373010b75caee1ee321864e960fc83594772497f95bfeab15786", + "h100-cancellation.json": "2088317c29cc9de80540f215f54b7d6d660044a568c1f50f6194449b2c9b4fba", + "h100-random.json": "66fcd857d26a57ea03ce0984f8bf20db1785336c61d302dce74dfd6a4cf213fb", + "h100-swiglu-like.json": "dcc64cf4360bbb9dca0211319f635cffed1cd91ebc6ddfd24052882a7201ba9d", + "h100-replays.json": "ee532d0a128eed4e978070fcc9fc6358c0cc0b72eafab50266c4645927cb438c" + }, + "transformation": "Aggregate all primary checks; preserve every primary/native failed check, both replay comparisons and four representative random timing cases. Omit repetitive traces, remaining timing samples, raw logs and execution identifiers; full originals retained privately. Private integrity does not establish public reproducibility." + }, + "artifacts": { + "h100-summary": { + "format": "json", + "original_sha256": "3a44e25a9397373010b75caee1ee321864e960fc83594772497f95bfeab15786", + "content": { + "historical_execution_date": "2026-10-10", + "rerun_for_this_edit": false, + "acceptance_complete": false, + "backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "device": { + "gpu": "NVIDIA H100 80GB HBM3", + "capability": [ + 9, + 0 + ], + "count": 1 + }, + "software": { + "python": "3.10.12", + "torch": "2.6.0+cu124", + "torch_cuda": "12.4", + "contract_version": "ws1-c1-v2", + "contract_sha256": "044167ec21e372017a2758dc09f805b368283c06b83f64061877628cbd101532" + }, + "primary_triton": "3.2.0", + "tested_source": { + "git_commit": "32b765ec992cd1206517104ec66506881203c91c", + "git_diff_sha256": "dfccc37be1ebdc3c62c0ed6ba89af360a67100248b3e2497636e23d162bff679", + "source_hashes": { + "csrc/cuda/gemm/det_gemm_kernel.cu": "08d08b0a32900fd9590d47f04819b34a2d37cbf53996aa5ed1373d88eae29f33", + "csrc/cuda/gemm/det_gemm_tma.cuh": "97049653e61f7eecfbb5807bd456cf04d27bcee810e382d176974f6cb28ba28d", + "rl_engine/kernels/ops/h3_ffn_down.py": "dc929c0b4b17133f2a9085781e20c5f83c3f66e1008ede35e9ef659ed4c7dd78", + "rl_engine/kernels/ops/h3_down_fp32_backend.py": "b2485d3135b4ac63163cf0f0786c0295da54efd413343171339a52814364f9bb", + "rl_engine/kernels/ops/triton/matmul/det_gemm.py": "b618a3e38223ab9bfcf3f644581f48f00cc48b50ddc0d772205aac4f6a5e37fe", + "rl_engine/kernels/ops/h3_down_validation.py": "967dd239a2daf2334ec134b3ac277fbdbd1de4a7244c0549a2f8b507ff537fbf", + "rl_engine/kernels/ops/canonical_backward.py": "0c443a9453d43bd16f3de72ccf9c681deb4c34088b18dbe199f7288227a14515", + "rl_engine/kernels/registry.py": "bb9ea616c66c0766eb09752d6e892ce9bebb3c68cac93672e918b34060ac2cf0", + "scripts/validate_h3_ffn_down.py": "ab394967074aff207f89177f1bafc4c24223187700d7910f355e6c92e3e0b772" + } + }, + "seed": 420, + "repeats": 5, + "dtype": "bfloat16", + "shapes": { + "input": "[M,14336]", + "weight": "[5376,14336]", + "output": "[M,5376]", + "rows_by_family": { + "cancellation": [ + 1, + 31, + 32, + 33, + 127, + 128, + 129, + 256, + 513, + 1024, + 2048, + 4096, + 8192, + 16385, + 24577, + 32768 + ], + "checkpoint_fixture": [ + 129 + ], + "random": [ + 1, + 31, + 32, + 33, + 127, + 128, + 129, + 256, + 513, + 1024, + 2048, + 4096, + 8192, + 16385, + 24577, + 32768 + ], + "swiglu_like": [ + 1, + 31, + 32, + 33, + 127, + 128, + 129, + 256, + 513, + 1024, + 2048, + 4096, + 8192, + 16385, + 24577, + 32768 + ] + } + }, + "fixture": { + "sha256": "63b2995505504ef0e91a66bb523c65c6d81b5be921005a9a9aca590587ba85fe", + "metadata": { + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "model_variant": "transformer", + "weight_key": "transformer_blocks.0.ff.net.2.weight", + "weight_source": "https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/42ed227ee7df40d41602854ae760620d6eb651fe/transformer/diffusion_pytorch_model-00001-of-00014.safetensors", + "weight_sha256": "e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f", + "index_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e", + "header_sha256": "74df10c38f82fe10b5d5983029684fb143f935032f52dc585c9c0ac9eff91746", + "activation_source": "seeded synthetic FP32 SiLU(gate) * up, then BF16", + "activation_kind": "synthetic", + "seed": 420 + } + }, + "gpu_tests": { + "status": "passed", + "passed": 64, + "command": "python -m pytest -q tests/test_h3_ffn_down.py tests/test_h3_down_tooling.py tests/test_h3_down_fixture.py tests/test_tolerance_contract.py", + "original_stdout_sha256": "89b85664fcf2fc6e0dc519e6656d0dddfe00c006ed98731ff2d9f126aba52c86" + }, + "matrix": { + "cases": 49, + "checks": 1141, + "passed_checks": 1141, + "accuracy": { + "y": { + "comparisons": 49, + "passed": true, + "max_abs_error": 0.12514114379882812, + "max_error_over_tolerance": 0.17978119850158691, + "failing_elements": 0, + "tolerance": { + "judgment": "forward_accuracy", + "op_class": "reduction", + "dtype_name": "bfloat16", + "status": "applicable", + "mode": "tolerance", + "atol": 0.05, + "rtol": 0.02, + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference", + "backend_profile": "cuda_bf16", + "arch_key": "sm90" + } + }, + "dx": { + "comparisons": 49, + "passed": true, + "max_abs_error": 0.015633583068847656, + "max_error_over_tolerance": 0.08594760298728943, + "failing_elements": 0, + "tolerance": { + "judgment": "gradient_accuracy", + "op_class": "reduction", + "dtype_name": "bfloat16", + "status": "applicable", + "mode": "tolerance", + "atol": 0.1, + "rtol": 0.02, + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference", + "backend_profile": "cuda_bf16", + "arch_key": "sm90" + } + }, + "dw": { + "comparisons": 49, + "passed": true, + "max_abs_error": 0.2525482177734375, + "max_error_over_tolerance": 0.18232759833335876, + "failing_elements": 0, + "tolerance": { + "judgment": "gradient_accuracy", + "op_class": "reduction", + "dtype_name": "bfloat16", + "status": "applicable", + "mode": "tolerance", + "atol": 0.1, + "rtol": 0.02, + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference", + "backend_profile": "cuda_bf16", + "arch_key": "sm90" + } + } + }, + "invariance": { + "checks": 994, + "passed": true, + "mismatched_bytes": 0, + "transforms": [ + "batch2_reshape", + "chunk31_single_backward", + "inactive_padding", + "other_rows_mutated", + "permuted_logical_rows", + "repeat", + "singleton_vs_batch", + "strided_input_and_weight" + ] + }, + "failed_checks": [] + }, + "timing_method": "Five-sample synchronized wall time including wrapper checks, copies, allocations and canonical preparation. Ordinary torch BF16 baseline is not qualified for byte invariance. Representative random rows only; all samples preserved privately.", + "representative_benchmarks": [ + { + "rows": 1, + "input_family": "random", + "benchmarks": { + "candidate_forward": { + "median_ms": 0.4641115665435791, + "samples_ms": [ + 0.5014799535274506, + 0.4641115665435791, + 0.4806891083717346, + 0.46247243881225586, + 0.46306103467941284 + ], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.19700080156326294, + "samples_ms": [ + 0.19984692335128784, + 0.21108612418174744, + 0.19700080156326294, + 0.18755346536636353, + 0.18952041864395142 + ], + "peak_allocated_bytes": 221347840, + "incremental_peak_bytes": 28672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.16470998525619507, + "samples_ms": [ + 0.18192827701568604, + 0.1676119863986969, + 0.16402080655097961, + 0.16470998525619507, + 0.1600421965122223 + ], + "peak_allocated_bytes": 376721408, + "incremental_peak_bytes": 155402240, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 5.881074815988541, + "samples_ms": [ + 10.020162910223007, + 4.396550357341766, + 4.599045962095261, + 5.881074815988541, + 7.008261978626251 + ], + "peak_allocated_bytes": 531011584, + "incremental_peak_bytes": 309692416, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.08215755224227905, + "samples_ms": [ + 0.107593834400177, + 0.08467957377433777, + 0.08215755224227905, + 0.07981061935424805, + 0.07973238825798035 + ], + "peak_allocated_bytes": 221329920, + "incremental_peak_bytes": 10752, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.2091360986232758, + "samples_ms": [ + 0.8774884045124054, + 1.2091360986232758, + 1.040332019329071, + 1.277294009923935, + 1.5825442969799042 + ], + "peak_allocated_bytes": 375499264, + "incremental_peak_bytes": 154180096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + } + }, + { + "rows": 129, + "input_family": "random", + "benchmarks": { + "candidate_forward": { + "median_ms": 0.5118474364280701, + "samples_ms": [ + 0.5456544458866119, + 0.5125962197780609, + 0.5012191832065582, + 0.5118474364280701, + 0.5014054477214813 + ], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 0.22323429584503174, + "samples_ms": [ + 0.2366490662097931, + 0.22323429584503174, + 0.21626055240631104, + 0.22421032190322876, + 0.21079927682876587 + ], + "peak_allocated_bytes": 234200576, + "incremental_peak_bytes": 4161024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 0.2612210810184479, + "samples_ms": [ + 0.29610469937324524, + 0.2780333161354065, + 0.25999173521995544, + 0.2612210810184479, + 0.24727359414100647 + ], + "peak_allocated_bytes": 390986240, + "incremental_peak_bytes": 160946688, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 4.367075860500336, + "samples_ms": [ + 9.567219763994217, + 4.193499684333801, + 4.239816218614578, + 4.367075860500336, + 4.3790265917778015 + ], + "peak_allocated_bytes": 564981248, + "incremental_peak_bytes": 334941696, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 0.11000409722328186, + "samples_ms": [ + 0.122804194688797, + 0.11217594146728516, + 0.10993704199790955, + 0.10891258716583252, + 0.11000409722328186 + ], + "peak_allocated_bytes": 231426560, + "incremental_peak_bytes": 1387008, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 1.257181167602539, + "samples_ms": [ + 1.0933689773082733, + 1.257181167602539, + 1.3633482158184052, + 1.405365765094757, + 0.9644664824008942 + ], + "peak_allocated_bytes": 389265920, + "incremental_peak_bytes": 159226368, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + } + }, + { + "rows": 8192, + "input_family": "random", + "benchmarks": { + "candidate_forward": { + "median_ms": 7.1127749979496, + "samples_ms": [ + 7.472507655620575, + 7.3508769273757935, + 7.102977484464645, + 7.1127749979496, + 6.655432283878326 + ], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 4.442032426595688, + "samples_ms": [ + 4.521660506725311, + 4.305578768253326, + 4.4516921043396, + 4.442032426595688, + 4.309028387069702 + ], + "peak_allocated_bytes": 1014235136, + "incremental_peak_bytes": 234881024, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 4.242591559886932, + "samples_ms": [ + 4.133425652980804, + 4.224028438329697, + 4.242591559886932, + 4.332114011049271, + 4.297386854887009 + ], + "peak_allocated_bytes": 933494784, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 18.447231501340866, + "samples_ms": [ + 18.785148859024048, + 18.268324434757233, + 18.301360309123993, + 19.21449601650238, + 18.447231501340866 + ], + "peak_allocated_bytes": 2291867648, + "incremental_peak_bytes": 1512513536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 1.7193928360939026, + "samples_ms": [ + 1.7297640442848206, + 1.718033105134964, + 1.7193928360939026, + 1.7182528972625732, + 1.7548836767673492 + ], + "peak_allocated_bytes": 867434496, + "incremental_peak_bytes": 88080384, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 5.222029983997345, + "samples_ms": [ + 5.2265264093875885, + 5.222029983997345, + 5.225643515586853, + 5.210563540458679, + 5.208142101764679 + ], + "peak_allocated_bytes": 1256456192, + "incremental_peak_bytes": 477102080, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + } + }, + { + "rows": 32768, + "input_family": "random", + "benchmarks": { + "candidate_forward": { + "median_ms": 41.62594676017761, + "samples_ms": [ + 41.61025583744049, + 41.62594676017761, + 41.63192957639694, + 41.6228361427784, + 41.692305356264114 + ], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dx": { + "median_ms": 35.47533228993416, + "samples_ms": [ + 33.461909741163254, + 34.31571647524834, + 37.22900524735451, + 35.47533228993416, + 35.92732176184654 + ], + "peak_allocated_bytes": 3393191936, + "incremental_peak_bytes": 939524096, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_dw_canonical_order": { + "median_ms": 19.52551305294037, + "samples_ms": [ + 19.52551305294037, + 19.28829774260521, + 17.499323934316635, + 20.080339163541794, + 20.596228539943695 + ], + "peak_allocated_bytes": 2607808512, + "incremental_peak_bytes": 154140672, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "candidate_forward_backward": { + "median_ms": 107.59353637695312, + "samples_ms": [ + 109.32975634932518, + 107.23195970058441, + 106.83485865592957, + 107.59353637695312, + 112.38069087266922 + ], + "peak_allocated_bytes": 7578877952, + "incremental_peak_bytes": 5125210112, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward": { + "median_ms": 7.167089730501175, + "samples_ms": [ + 6.37710839509964, + 6.479129195213318, + 7.427748292684555, + 7.846798747777939, + 7.167089730501175 + ], + "peak_allocated_bytes": 2805989376, + "incremental_peak_bytes": 352321536, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + }, + "torch_bf16_forward_backward": { + "median_ms": 21.537311375141144, + "samples_ms": [ + 22.03044295310974, + 21.537311375141144, + 20.823486149311066, + 21.31655439734459, + 21.96405827999115 + ], + "peak_allocated_bytes": 3899654144, + "incremental_peak_bytes": 1445986304, + "timing_scope": "synchronized wall time, includes wrapper/copies/padding" + } + } + } + ], + "trace_contract": { + "requested_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "actual_backend": "rlkernel.h3_ffn_down.triton.fp32_tiles.v1", + "kernel_id": "rl_engine.kernels.ops.triton.matmul.det_gemm._det_gemm_fp32_kernel", + "fallback": false, + "arithmetic": "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast", + "tile": [ + 64, + 64, + 32 + ], + "num_warps": 4, + "num_stages": 2, + "fp_fusion": false, + "split_k": false, + "split_policy": "none", + "accumulator_dtype": "float32", + "schedule_id": "fixed-64x64x32-w4-s2-fusion0", + "topology": { + "world_size": 1, + "tensor_parallel_size": 1 + }, + "checkpoint_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "weight_layout": "[5376,14336]", + "dw_order": "lexicographic active logical keys; trailing zero pad to 32", + "forward_executed": true, + "input_gradient_executed": true, + "weight_gradient_executed": true + }, + "all_primary_traces_use_requested_backend_without_fallback": true, + "rejected_native": { + "backend": "rlkernel.h3_ffn_down.sm90.midtree.v1", + "status": "failed", + "failed_checks": [ + { + "rows": 129, + "input_family": "checkpoint_fixture", + "judgment": "forward_accuracy", + "tolerance": { + "judgment": "forward_accuracy", + "op_class": "reduction", + "dtype_name": "bfloat16", + "status": "applicable", + "mode": "tolerance", + "atol": 0.05, + "rtol": 0.02, + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference", + "backend_profile": "cuda_bf16", + "arch_key": "sm90" + }, + "passed": false, + "max_abs_error": 0.40984344482421875, + "max_error_over_tolerance": 5.004410743713379, + "failing_elements": 23353, + "tensor": "y", + "transform": "fp32_reference" + } + ] + }, + "limitations": [ + "Real checkpoint weights were paired with synthetic activations, not captured H3 activations.", + "One SM90 GPU, eager WS1 only; no whole-model or TP qualification.", + "The default fixed-tile adapter is slower than the ordinary PyTorch BF16 comparator.", + "The vLLM probe does not qualify canonical autograd or all dW invariance transforms.", + "Strict production resolution remains gated by the test-only descriptor.", + "Independent hardware/environment replay, captured activations and production runtime qualification remain pending.", + "No GPU rerun after source formatting, directory relocation or registry merges." + ], + "replay": { + "cold_cache_output_comparison": { + "status": "passed", + "scope": "Same H100 and PyTorch profile; separate process and fresh Triton 3.2.0 cache; not another GPU or runtime environment", + "checks": [ + { + "input_family": "random", + "rows": 1, + "tensor": "y", + "passed": true, + "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386" + }, + { + "input_family": "random", + "rows": 1, + "tensor": "dx", + "passed": true, + "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f" + }, + { + "input_family": "random", + "rows": 1, + "tensor": "dw", + "passed": true, + "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9" + }, + { + "input_family": "random", + "rows": 33, + "tensor": "y", + "passed": true, + "sha256": "250f624bedafbd8468babdf59a1c3320e1047b7def946d1c97a399b4ce76e869" + }, + { + "input_family": "random", + "rows": 33, + "tensor": "dx", + "passed": true, + "sha256": "652ffe9aa3ee3dadb5a0fcb3447a42e1b15b5c50442e2b13332f8722ddac04ee" + }, + { + "input_family": "random", + "rows": 33, + "tensor": "dw", + "passed": true, + "sha256": "ad736ee7cffc7352d2ca352ac21a7c8db4a65cf967ccb93893eacd1196a25a08" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078" + }, + { + "input_family": "swiglu_like", + "rows": 1, + "tensor": "y", + "passed": true, + "sha256": "45c0a3d8ac89ad6e6d9abf0c3b651f78aa29d21a0658d962621ab80d0c8f64eb" + }, + { + "input_family": "swiglu_like", + "rows": 1, + "tensor": "dx", + "passed": true, + "sha256": "afe5c6f16746123e0e15fa11293826ce8ba4d409eb9ee795182e68f7032f2e43" + }, + { + "input_family": "swiglu_like", + "rows": 1, + "tensor": "dw", + "passed": true, + "sha256": "5fda4991a478f32e87fd328ed21717c14c89afaa575d6817065043e41e76beda" + }, + { + "input_family": "swiglu_like", + "rows": 33, + "tensor": "y", + "passed": true, + "sha256": "5ff078a836c628189018465b2f3df6f28325ef6ebf48d75d85679d5fb3d3400e" + }, + { + "input_family": "swiglu_like", + "rows": 33, + "tensor": "dx", + "passed": true, + "sha256": "9a51175a297f58b4e1954b0c7d23905161a6f9ef4c8dc8dec95ee657d4c51c60" + }, + { + "input_family": "swiglu_like", + "rows": 33, + "tensor": "dw", + "passed": true, + "sha256": "ac7a1a891b5401b1f26645394532ff4c9556f48ea786155ffc02af067071f69f" + }, + { + "input_family": "swiglu_like", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "6f18c30644e43ba2934c76eb8bcfefc298c69935baa26bc7d92e28cf029dc236" + }, + { + "input_family": "swiglu_like", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "ec72b2ebce5759c4bbab75b88d698c3b69e26408cb8394517f7a9f66e1ed4bf8" + }, + { + "input_family": "swiglu_like", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "1b66c82f7aced61c44f6e0b7769a66ec60116a35856af26388ce21ed4efc593f" + }, + { + "input_family": "cancellation", + "rows": 1, + "tensor": "y", + "passed": true, + "sha256": "7ffd3684b2dec669090662fe5fc98e32ef0908b275d5e2af5ded586e10860421" + }, + { + "input_family": "cancellation", + "rows": 1, + "tensor": "dx", + "passed": true, + "sha256": "693f69f6ffb23114cc40f559e78d3f0c96fc7edaff65bcd3c7f0cae702c87730" + }, + { + "input_family": "cancellation", + "rows": 1, + "tensor": "dw", + "passed": true, + "sha256": "0075000b70889eba017ba0c596e3072654b517819d2ae04e814e2cf1b8b63aa3" + }, + { + "input_family": "cancellation", + "rows": 33, + "tensor": "y", + "passed": true, + "sha256": "6bcd40750594f0bc7db22a4c79e3d1f2a3ff201531f34fddef6bf2538b24362b" + }, + { + "input_family": "cancellation", + "rows": 33, + "tensor": "dx", + "passed": true, + "sha256": "25cce04e11335a6a017a7455cce810dbc2b2815cedb877ae63ab803d7ad9ede3" + }, + { + "input_family": "cancellation", + "rows": 33, + "tensor": "dw", + "passed": true, + "sha256": "77c8a96db1a6712311503033033dccd73eb1aab761cbc4f0bdab404596c34f18" + }, + { + "input_family": "cancellation", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "c020992d561b7a21ac8f25417270656ba5ca4ba5295f2b4ef2804c2d3ea39ea9" + }, + { + "input_family": "cancellation", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "6f3bbc1b79f7fd84c4d8c0cf5f1bacd36aee6425162ece92a543eba5aecad930" + }, + { + "input_family": "cancellation", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "f602eb2cb919c47c1055fa23105171bd12cf0d8f0fec348e5d8cfc0f0ee48716" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41" + } + ] + }, + "compiler_version_output_comparison": { + "status": "passed", + "scope": "Same H100/PyTorch; Triton 3.2.0 versus separately pinned 3.4.0 using independent JIT caches; selected cases only", + "checks": [ + { + "input_family": "random", + "rows": 1, + "tensor": "y", + "passed": true, + "sha256": "e5fe21151c0dd93aa2b06c9f1741f1d890bff8b6033cb3254f02caebc6315386" + }, + { + "input_family": "random", + "rows": 1, + "tensor": "dx", + "passed": true, + "sha256": "a88ab71d4c7c1f8f42501065d445aeea13d06141dc574a9a664a06d2191aa81f" + }, + { + "input_family": "random", + "rows": 1, + "tensor": "dw", + "passed": true, + "sha256": "82679e4fc3d45760a6506b51e690bc7c085e7eb9ab533af941cc3bea52d6def9" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "09b874683cd430aef29e88c69c6045e14550c43d0b5ea8e2d10e4c0b83f1bcb7" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "5e9612df14e42ef4c29d0a87dc0e9c5a2e52538febe0833b1e84404d2f591a5d" + }, + { + "input_family": "random", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "189c4f5e27489c154533f2b6ff7938cea2a78d928fbf46054a0a58963c15e078" + }, + { + "input_family": "random", + "rows": 8192, + "tensor": "y", + "passed": true, + "sha256": "5f6470e031a61422734ca646ab827a65b5a1031eba2c7153e7bde3c7e36e70c8" + }, + { + "input_family": "random", + "rows": 8192, + "tensor": "dx", + "passed": true, + "sha256": "6d0ac2f7b0b0116d5cb8baf59e6b78b5d1bb77da15ae3631a9b52ee74a73ea66" + }, + { + "input_family": "random", + "rows": 8192, + "tensor": "dw", + "passed": true, + "sha256": "7ff5fd6196633fee5ec0539a919f7725249c7389eb3a74d669be1b2095b31328" + }, + { + "input_family": "random", + "rows": 32768, + "tensor": "y", + "passed": true, + "sha256": "2887e2261ca6bed9765a1531db63ef2e6f2dcd46a20bedc18ff1dd4a0ebf8949" + }, + { + "input_family": "random", + "rows": 32768, + "tensor": "dx", + "passed": true, + "sha256": "b57014e1a7a27923da784c6b0ceef1bf541f24e768a0a5224c6859bc506c0515" + }, + { + "input_family": "random", + "rows": 32768, + "tensor": "dw", + "passed": true, + "sha256": "fc5f279f4dd80d4995540573c5c6a1a9feb33341f33f4586f599dc2c23e4d42b" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "y", + "passed": true, + "sha256": "b459b3edbe9e388b9270541cbc9764a7315f74712c08075878936d8d1907186b" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "dx", + "passed": true, + "sha256": "082851ec9fd8976b500b318670a202f0f5431fe2042799f6db2530fca04c2e84" + }, + { + "input_family": "checkpoint_fixture", + "rows": 129, + "tensor": "dw", + "passed": true, + "sha256": "8b533d8735e80018237b64cbc0c0b0cbaffb66d276b68815c05b7a5a9f1c3b41" + } + ] + } + }, + "prior_art": "The vLLM partial probe requires Triton 3.4.0; Triton 3.2.0 rejected flatten=True. It does not qualify canonical autograd or all dW invariance transforms; timings are not end-to-end comparable." + } + } + } +} diff --git a/docs/operators/h3-ffn-down-results.sha256 b/docs/operators/h3-ffn-down-results.sha256 new file mode 100644 index 000000000..3dabed029 --- /dev/null +++ b/docs/operators/h3-ffn-down-results.sha256 @@ -0,0 +1 @@ +c1b8cc36a9cc8cb75891517f17f80736e751ba0bb4fc30ef71e987c53e4d8d9c h3-ffn-down-results.json diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index c97049231..00b717b5e 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -115,59 +115,28 @@ Within 3.4.0 at random M=32768, vLLM forward took 9.6604 ms versus this adapter' The vLLM primitive excludes this adapter's checks/copies and has no qualified canonical autograd session or full dW invariance matrix. It remains prior art. -## Core JSON evidence - -The following sanitized historical JSON bundles accompany this document: - -- [Summary, fixture, tests and rejected native candidate](evidence/h3-down-h100-20261010/h100-summary.json). -- Full matrix: [random](evidence/h3-down-h100-20261010/h100-random.json), - [SwiGLU-like](evidence/h3-down-h100-20261010/h100-swiglu-like.json), and - [cancellation](evidence/h3-down-h100-20261010/h100-cancellation.json). -- [Cold-cache/compiler replay and vLLM comparison](evidence/h3-down-h100-20261010/h100-replays.json). - -Read each report from `artifacts[filename].content`. The split matrix is lossless: combine the three `cases` arrays with the summary's -`fixture_cases`, sort by their `case_indexes` / `fixture_case_indexes`, and attach -`full_report_header`. That reproduces the measured report with portable execution paths. -Every file is below the repository's 500 KiB pre-commit limit. JSON retains all -checks, tolerances, timing samples, peak memory, output hashes, compiled-kernel -identities and source fingerprints. Original logs, PNG/CSV, source snapshots, -archives and original execution metadata remain local and are excluded from the PR. - -GPU reports identify the measured source. Subsequent Black/isort formatting -is recorded separately with original/current hashes, unchanged non-import AST -and unchanged import bindings; no GPU rerun after formatting is claimed. - -The target-branch merge relocates the operator, adapter and tests to canonical -packages and registers the test-only descriptor in `rl_engine.runtime.registry`. -New reports hash the actual canonical device kernel, autograd runtime and registry; -compatibility shims are not used as implementation fingerprints. The device -function and operator/session non-import AST are unchanged from the submitted -candidate. The original five measured JSON files were preserved unchanged outside the submission. -Each submitted bundle records its original SHA256 and source commit; only concrete -execution paths were replaced with portable placeholders. All measurements, failures -and limitations are retained. `SHA256SUMS` pins the sanitized bundles. Local -post-merge regression passed **101 tests**, with **3 SM90-only tests skipped**; -the upstream layout/build suite additionally passed **32 tests** with **1 skipped**. -No GPU rerun after the directory merge is claimed. GitHub Actions CPU/GPU workflows -do not target `test-h3`; local checks and archived H100 runs are separate evidence. - -The subsequent merge of `test-h3@0e8a18a` retains both the H3 down projection -and upstream video packing registry descriptors. H3 operator, adapter, device -kernel and autograd implementations are unchanged from the previous PR head. -Local regression after this conflict resolution passed **156 tests**, with -**3 SM90-only tests skipped** on a machine without a usable GPU. The plotting -check passed after installing its missing local dependencies. Reproduce the suite: - -```bash -python -m pytest -q tests/ops/gemm/test_h3_ffn_down.py \ - tests/validation/operators/test_h3_down_fixture.py \ - tests/validation/operators/test_h3_down_tooling.py \ - tests/runtime/test_kernel_registry.py tests/runtime/test_layout_compatibility.py \ - tests/build/test_build_platform_collectives.py \ - tests/ops/packing/test_video_patch_pack_unpack.py -``` - -No new GPU validation is claimed for this registry conflict resolution. +## Evidence and implementation inventory + +[Compact historical results](h3-ffn-down-results.json) retain the complete matrix +summary, forward/dX/dW tolerances and worst errors, replay digests, representative +timings, rejected native results and original bundle hashes. Full reports, samples, +traces and raw originals are preserved privately. This is a selected derivative, +not a lossless report or a new GPU run. Historical published copies remain in Git +history; private archives do not establish public reproducibility. + +- Operator: `rl_engine/ops/gemm/h3_ffn_down.py`. +- Triton adapter: `rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py`; + actual device computation reuses `_det_gemm_fp32_kernel` in `det_gemm.py`. +- Canonical gradients: `rl_engine/ops/autograd/canonical_backward.py`. +- Native CUDA comparator: existing SM90 midpoint-tree backend; rejected for the + accuracy failure above, never an automatic fallback. +- Tests: `tests/ops/gemm/test_h3_ffn_down.py` and + `tests/validation/operators/test_h3_down_{fixture,tooling}.py`. + +Historical local checks at `d8ae6cf` passed 58 tests with 13 GPU-dependent skips +(registry, full-attention and H3 down selections). No tests or GPU runs were +repeated for this documentation/evidence reduction. Numerical sources are unchanged; +source fingerprints and derivative values are checked separately from test execution. ## Reproduction and remaining qualification From 3d89140b1c10c3741638d1a86ab47305177cc8fe Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:37:12 +0800 Subject: [PATCH 04/10] Move H3 down projection tools to canonical directories Update test imports, repository-root discovery, runner commands and the reproduction guide for the repository layout. Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- .../operators/gemm}/plot_h3_down_report.py | 0 {scripts => ci/scripts}/run_h3_down_h100.sh | 8 ++++---- docs/operators/h3-ffn-down.md | 8 ++++---- .../requirements-h3-down.txt => requirements/h3-down.txt | 0 tests/validation/operators/test_h3_down_fixture.py | 2 +- tests/validation/operators/test_h3_down_tooling.py | 6 +++--- .../validation/operators}/validate_h3_ffn_down.py | 4 ++-- {scripts => tools/weights}/prepare_h3_down_fixture.py | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) rename {scripts => benchmarks/operators/gemm}/plot_h3_down_report.py (100%) rename {scripts => ci/scripts}/run_h3_down_h100.sh (88%) rename scripts/requirements-h3-down.txt => requirements/h3-down.txt (100%) rename {scripts => tools/validation/operators}/validate_h3_ffn_down.py (99%) rename {scripts => tools/weights}/prepare_h3_down_fixture.py (99%) diff --git a/scripts/plot_h3_down_report.py b/benchmarks/operators/gemm/plot_h3_down_report.py similarity index 100% rename from scripts/plot_h3_down_report.py rename to benchmarks/operators/gemm/plot_h3_down_report.py diff --git a/scripts/run_h3_down_h100.sh b/ci/scripts/run_h3_down_h100.sh similarity index 88% rename from scripts/run_h3_down_h100.sh rename to ci/scripts/run_h3_down_h100.sh index 90193f402..3dade2bb0 100644 --- a/scripts/run_h3_down_h100.sh +++ b/ci/scripts/run_h3_down_h100.sh @@ -3,10 +3,10 @@ # Copyright (c) 2026 RL-Kernel Contributors set -euo pipefail -repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo_dir" mode="${1:-smoke}" -case "$mode" in smoke|full) ;; *) echo 'Usage: bash scripts/run_h3_down_h100.sh [smoke|full]' >&2; exit 2 ;; esac +case "$mode" in smoke|full) ;; *) echo 'Usage: bash ci/scripts/run_h3_down_h100.sh [smoke|full]' >&2; exit 2 ;; esac result_dir="${H3_RESULT_DIR:-$repo_dir/artifacts/h3-down/$(date -u +%Y%m%dT%H%M%SZ)}" mkdir -p "$result_dir" export KERNEL_ALIGN_USE_FAST_MATH=0 @@ -69,10 +69,10 @@ fi if [[ -n "${H3_FIXTURE:-}" ]]; then args+=(--fixture "$H3_FIXTURE"); fi if [[ "${H3_COMPARE_VLLM:-0}" == 1 ]]; then args+=(--compare-vllm); fi set +e -python scripts/validate_h3_ffn_down.py "${args[@]}" 2>&1 | tee "$result_dir/validation.log" +python tools/validation/operators/validate_h3_ffn_down.py "${args[@]}" 2>&1 | tee "$result_dir/validation.log" validation_status=${PIPESTATUS[0]} set -e -python scripts/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir" +python benchmarks/operators/gemm/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir" echo "Evidence directory: $result_dir" if [[ "$gpu_test_status" -ne 0 ]]; then exit "$gpu_test_status"; fi exit "$validation_status" diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index 00b717b5e..700570806 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -146,10 +146,10 @@ Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: python3.10 -m venv .venv source .venv/bin/activate python -m pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 -python -m pip install -r scripts/requirements-h3-down.txt -python scripts/prepare_h3_down_fixture.py --rows 129 --output ${FIXTURE_ROOT}/h3-down-fixture.pt -H3_FIXTURE=${FIXTURE_ROOT}/h3-down-fixture.pt bash scripts/run_h3_down_h100.sh smoke -python scripts/validate_h3_ffn_down.py --seed 420 --repeats 5 \ +python -m pip install -r requirements/h3-down.txt +python tools/weights/prepare_h3_down_fixture.py --rows 129 --output ${FIXTURE_ROOT}/h3-down-fixture.pt +H3_FIXTURE=${FIXTURE_ROOT}/h3-down-fixture.pt bash ci/scripts/run_h3_down_h100.sh smoke +python tools/validation/operators/validate_h3_ffn_down.py --seed 420 --repeats 5 \ --rows 1,31,32,33,127,128,129,256,513,1024,2048,4096,8192,16385,24577,32768 \ --families random,swiglu_like,cancellation \ --fixture ${FIXTURE_ROOT}/h3-down-fixture.pt --output /tmp/h3-down-full.json diff --git a/scripts/requirements-h3-down.txt b/requirements/h3-down.txt similarity index 100% rename from scripts/requirements-h3-down.txt rename to requirements/h3-down.txt diff --git a/tests/validation/operators/test_h3_down_fixture.py b/tests/validation/operators/test_h3_down_fixture.py index a88f5fd81..32150f71b 100644 --- a/tests/validation/operators/test_h3_down_fixture.py +++ b/tests/validation/operators/test_h3_down_fixture.py @@ -9,7 +9,7 @@ spec = importlib.util.spec_from_file_location( "h3_fixture_under_test", - Path(__file__).resolve().parents[3] / "scripts/prepare_h3_down_fixture.py", + Path(__file__).resolve().parents[3] / "tools/weights/prepare_h3_down_fixture.py", ) fixture = importlib.util.module_from_spec(spec) spec.loader.exec_module(fixture) diff --git a/tests/validation/operators/test_h3_down_tooling.py b/tests/validation/operators/test_h3_down_tooling.py index 569d46978..e5f4e58bb 100644 --- a/tests/validation/operators/test_h3_down_tooling.py +++ b/tests/validation/operators/test_h3_down_tooling.py @@ -16,7 +16,7 @@ def test_gpu_validator_refuses_cpu_and_preserves_error_evidence(tmp_path, monkey import torch spec = importlib.util.spec_from_file_location( - "h3_validator_under_test", ROOT / "scripts/validate_h3_ffn_down.py" + "h3_validator_under_test", ROOT / "tools/validation/operators/validate_h3_ffn_down.py" ) validator = importlib.util.module_from_spec(spec) spec.loader.exec_module(validator) @@ -59,7 +59,7 @@ def test_failed_run_renders_summary_without_inventing_charts(tmp_path): result = subprocess.run( [ sys.executable, - str(ROOT / "scripts/plot_h3_down_report.py"), + str(ROOT / "benchmarks/operators/gemm/plot_h3_down_report.py"), str(source), "--output-dir", str(tmp_path), @@ -112,7 +112,7 @@ def test_plot_pipeline_with_explicitly_synthetic_test_data(tmp_path): result = subprocess.run( [ sys.executable, - str(ROOT / "scripts/plot_h3_down_report.py"), + str(ROOT / "benchmarks/operators/gemm/plot_h3_down_report.py"), str(source), "--output-dir", str(tmp_path), diff --git a/scripts/validate_h3_ffn_down.py b/tools/validation/operators/validate_h3_ffn_down.py similarity index 99% rename from scripts/validate_h3_ffn_down.py rename to tools/validation/operators/validate_h3_ffn_down.py index 801303f9f..8f77acffb 100644 --- a/scripts/validate_h3_ffn_down.py +++ b/tools/validation/operators/validate_h3_ffn_down.py @@ -17,7 +17,7 @@ import traceback from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT)) import torch # noqa: E402 @@ -103,7 +103,7 @@ def provenance(args): "rl_engine/backends/extension.py", "rl_engine/contracts/numerical.py", "rl_engine/runtime/registry.py", - "scripts/validate_h3_ffn_down.py", + "tools/validation/operators/validate_h3_ffn_down.py", ): result["source_hashes"][name] = sha256(ROOT / name) if base._EXT_AVAILABLE: diff --git a/scripts/prepare_h3_down_fixture.py b/tools/weights/prepare_h3_down_fixture.py similarity index 99% rename from scripts/prepare_h3_down_fixture.py rename to tools/weights/prepare_h3_down_fixture.py index 90c4bdaa8..2077637f8 100644 --- a/scripts/prepare_h3_down_fixture.py +++ b/tools/weights/prepare_h3_down_fixture.py @@ -14,7 +14,7 @@ import urllib.request from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) import torch # noqa: E402 From aae5b903b8503bc87017283677128bf08d5f55d7 Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:34:20 +0800 Subject: [PATCH 05/10] Correct H3 down projection arithmetic and backward provenance Describe BF16 MMA with FP32 accumulation, retain one backend per forward, and distinguish gradient kernel roles. Prepare padding indices during forward while preserving strict padded-gradient validation. Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- docs/operators/h3-ffn-down.md | 37 +++++++-- .../triton/gemm/h3_down_fp32_backend.py | 11 ++- rl_engine/ops/gemm/h3_ffn_down.py | 51 +++++++----- rl_engine/runtime/registry.py | 2 +- tests/ops/gemm/test_h3_ffn_down.py | 77 +++++++++++++++++++ 5 files changed, 146 insertions(+), 32 deletions(-) diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index 700570806..ecb9955d6 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -10,9 +10,11 @@ Checkpoint: `MiniMaxAI/MiniMax-H3@42ed227ee7df40d41602854ae760620d6eb651fe`. Backend `rlkernel.h3_ffn_down.triton.fp32_tiles.v1` reuses the existing `rl_engine.backends.shared.triton.gemm.det_gemm._det_gemm_fp32_kernel`. -The historical logical kernel identifier in traces remains stable. BF16 operands feed FP32 -32-wide dot products and ascending FP32 tile additions, followed by one BF16 -cast. The schedule is fixed at 64x64x32, four warps and two stages, with +The historical backend, class and logical kernel identifiers remain stable; +`fp32_tiles` denotes accumulation precision, not FP32 multiplication. With +`PROMOTE_INPUTS=False`, BF16 operands feed 32-wide BF16 tensor-core MMA +tiles with FP32 accumulation and ascending FP32 tile additions, followed +by one BF16 cast. The schedule is fixed at 64x64x32, four warps and two stages, with `enable_fp_fusion=False`, TF32 disabled, no tuning, no split-K and no fallback. No new CUDA or Triton device kernel is added. @@ -61,7 +63,16 @@ and microbatching. A negative first component marks inactive padding. The same weight Tensor must retain one parameter ID. Do not sum independently rounded microbatch weight gradients; call `validate_complete()` to detect missing branches. Execution traces record requested/actual backend, kernel, arithmetic, schedule, -topology, checkpoint, executed gradients and actual JIT/cubin hashes. +topology, checkpoint, separate `input_gradient_kernel` / `weight_gradient_kernel` +fields and actual JIT/cubin hashes. A deferred canonical weight reduction keeps +its kernel field unset until that use executes the reduction. + +Each forward validates and constructs its backend once and retains it for +backward. Padding row indices are snapshotted during forward registration; +unpadded uses skip the padding-related backward readback. Padded uses still +synchronously reject nonzero upstream padding gradients. Forward registration +and canonical row preparation can still synchronize; this is not an +asynchronous execution guarantee or a measured speedup. ## H100 results @@ -122,7 +133,9 @@ summary, forward/dX/dW tolerances and worst errors, replay digests, representati timings, rejected native results and original bundle hashes. Full reports, samples, traces and raw originals are preserved privately. This is a selected derivative, not a lossless report or a new GPU run. Historical published copies remain in Git -history; private archives do not establish public reproducibility. +history; private archives do not establish public reproducibility. The historical +JSON retains its original FP32-dot trace wording; the corrected BF16 MMA / FP32 +accumulation description above applies to that unchanged device computation. - Operator: `rl_engine/ops/gemm/h3_ffn_down.py`. - Triton adapter: `rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py`; @@ -135,8 +148,18 @@ history; private archives do not establish public reproducibility. Historical local checks at `d8ae6cf` passed 58 tests with 13 GPU-dependent skips (registry, full-attention and H3 down selections). No tests or GPU runs were -repeated for this documentation/evidence reduction. Numerical sources are unchanged; -source fingerprints and derivative values are checked separately from test execution. +repeated for that documentation/evidence reduction. The subsequent review fix +corrects the arithmetic description and changes wrapper preparation and gradient +tracing, while preserving the device kernel, launch settings and canonical +reduction arithmetic. The H100 results above remain historical and were not +rerun for this fix; they do not measure the updated wrapper or establish its +GPU acceptance. Original reports, summary bytes and evidence pins are unchanged. + +Local review-fix checks on 2026-10-26: **56 passed, 3 skipped** across +`tests/ops/gemm/test_h3_ffn_down.py`, `tests/runtime/test_kernel_registry.py` +and `tests/runtime/test_layout_compatibility.py`. The three skips require SM90; +the new tests exercise CPU orchestration, not GPU arithmetic. Lint and formatting +checks passed. No GPU run or performance measurement was made for this fix. ## Reproduction and remaining qualification diff --git a/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py b/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py index d41848e2a..a35ffd7ac 100644 --- a/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py +++ b/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 RL-Kernel Contributors -"""H3 adapter for the existing fixed-order Triton FP32 tile GEMM kernel.""" +"""H3 BF16 adapter for the existing fixed-order Triton GEMM kernel.""" import hashlib @@ -10,9 +10,12 @@ class FixedFP32Backend: - """BF16 operands; FP32 dot tiles and ordered FP32 additions; one final cast. + """BF16 MMA tiles with FP32 accumulation and ordered FP32 tile additions. - Reuses `_det_gemm_fp32_kernel`, bypassing its FP32-input/output wrapper. + Reuses `_det_gemm_fp32_kernel` with PROMOTE_INPUTS=False and a BF16 output. + Operands stay BF16; each 32-wide dot accumulates in FP32 before the ordered + FP32 tile sum and one final BF16 cast. The class name is a historical alias + for the accumulator precision, not FP32 multiplication. Fixed 64x64x32 tiles, four warps, two stages, no tuning or split-K. """ @@ -22,7 +25,7 @@ def __init__(self): from rl_engine.backends.shared.triton.gemm import det_gemm if not det_gemm._TRITON_AVAILABLE: - raise RuntimeError("Triton is required for H3 FP32 tiles; fallback is forbidden") + raise RuntimeError("Triton is required for H3 BF16 MMA tiles; fallback is forbidden") self.kernel = det_gemm._det_gemm_fp32_kernel self.triton = det_gemm.triton diff --git a/rl_engine/ops/gemm/h3_ffn_down.py b/rl_engine/ops/gemm/h3_ffn_down.py index 90b3adf89..572f43b63 100644 --- a/rl_engine/ops/gemm/h3_ffn_down.py +++ b/rl_engine/ops/gemm/h3_ffn_down.py @@ -2,7 +2,7 @@ # Copyright (c) 2026 RL-Kernel Contributors """H3 BF16 down projection: an explicit SM90 reuse candidate for issue #420. -The default reuses the existing fixed-order FP32 Triton tile kernel. The native +The default reuses fixed-order BF16 Triton MMA tiles with FP32 accumulation. The native BF16 midpoint tree is retained as an explicit legacy comparison backend. """ @@ -104,7 +104,10 @@ def _register_rows(session, weight, keys, rows, parameter_id): raise ValueError("active logical keys must be unique across parameter uses") slot = session.register(parameter_id, keys) bindings[parameter_id] = weight - return slot, keys + # Resolve the dynamic padding selection during forward registration. + # Unpadded backward calls then need no padding-related device readback. + padding_rows = torch.nonzero(keys[:, 0] < 0, as_tuple=True)[0] + return slot, padding_rows def _weight_gradient(extension, rows, grads): @@ -123,15 +126,14 @@ def _weight_gradient(extension, rows, grads): class _H3DownFn(torch.autograd.Function): @staticmethod - def forward(ctx, x, weight, session, slot, keys, parameter_id, trace): - extension = _require_backend(x.device, trace["requested_backend"]) + def forward(ctx, x, weight, session, slot, padding_rows, parameter_id, trace, extension): flat = x.reshape(-1, H3_FFN_DIM).contiguous() native_weight = weight.contiguous() # The native extension uses the current device's stream and SM query. with torch.cuda.device(x.device): y = extension.det_gemm_fwd_rhs_transposed(flat, native_weight) # Retain original version counters even when layout normalization copies. - ctx.save_for_backward(flat, native_weight, keys, x, weight) + ctx.save_for_backward(flat, native_weight, padding_rows, x, weight) ctx.session, ctx.slot, ctx.parameter_id = session, slot, parameter_id ctx.extension, ctx.trace, ctx.input_shape = extension, trace, x.shape trace.update(actual_backend=trace["requested_backend"], forward_executed=True) @@ -145,18 +147,22 @@ def forward(ctx, x, weight, session, slot, keys, parameter_id, trace): @staticmethod @once_differentiable def backward(ctx, grad_output): - x, weight, keys, _original_x, _original_weight = ctx.saved_tensors + x, weight, padding_rows, _original_x, _original_weight = ctx.saved_tensors grad = grad_output.reshape(-1, H3_HIDDEN_DIM).contiguous() if grad.dtype != torch.bfloat16: raise TypeError("H3 upstream gradient must have dtype bfloat16") - if keys is not None and torch.any(grad[keys[:, 0] < 0] != 0).item(): - raise ValueError("padding rows must have zero upstream gradient") + if padding_rows is not None and padding_rows.numel(): + # Keep strict validation for padded uses; upstream zeroing is not guaranteed. + if torch.any(grad.index_select(0, padding_rows) != 0).item(): + raise ValueError("padding rows must have zero upstream gradient") dx = dw = None executed = [] with torch.cuda.device(x.device): if ctx.needs_input_grad[0]: dx = ctx.extension.det_gemm_fwd(grad, weight).reshape(ctx.input_shape) - executed.append(getattr(ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_fwd")) + kernel = getattr(ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_fwd") + executed.append(kernel) + ctx.trace["input_gradient_kernel"] = kernel ctx.trace["input_gradient_executed"] = True if hasattr(ctx.extension, "h3_last_launch"): ctx.trace["input_gradient_launch"] = ctx.extension.h3_last_launch @@ -164,11 +170,11 @@ def backward(ctx, grad_output): def reducer(rows, grads): value = _weight_gradient(ctx.extension, rows, grads) - executed.append( - getattr( - ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_db_transposed" - ) + kernel = getattr( + ctx.extension, "h3_kernel_id", "rl_engine._C.det_gemm_db_transposed" ) + executed.append(kernel) + ctx.trace["weight_gradient_kernel"] = kernel ctx.trace["weight_gradient_executed"] = True ctx.trace["canonical_active_rows"] = rows.shape[0] if hasattr(ctx.extension, "h3_last_launch"): @@ -179,11 +185,11 @@ def reducer(rows, grads): if executed: record_backward( "h3_ffn_down_gemm", - kernel_id="+".join(executed), + kernel_id="+".join(dict.fromkeys(executed)), impl=ctx.trace["requested_backend"], family="triton" if ctx.trace["requested_backend"] == H3_DOWN_BACKEND else "cuda", ) - return dx, dw, None, None, None, None, None + return dx, dw, None, None, None, None, None, None class H3FFNDownGemmOp: @@ -204,13 +210,13 @@ def __call__(self, x, weight, *, logical_keys=None, parameter_id="h3.ffn.down.we _validate_inputs(x, weight) if torch.compiler.is_compiling(): raise RuntimeError("this H3 profile currently supports eager execution only") - _require_backend(x.device, self.backend) - session, slot, keys = None, None, None + extension = _require_backend(x.device, self.backend) + session, slot, padding_rows = None, None, None if torch.is_grad_enabled() and weight.requires_grad: session = active_session() if session is None: raise RuntimeError("weight gradients require an active canonical backward session") - slot, keys = _register_rows( + slot, padding_rows = _register_rows( session, weight, logical_keys, x.numel() // H3_FFN_DIM, parameter_id ) trace = { @@ -219,7 +225,8 @@ def __call__(self, x, weight, *, logical_keys=None, parameter_id="h3.ffn.down.we "kernel_id": None, "fallback": False, "arithmetic": ( - "32-wide FP32 dot tiles, ordered FP32 additions, one BF16 epilogue cast" + "32-wide BF16 MMA tiles with FP32 accumulation, ordered FP32 additions, " + "one BF16 epilogue cast" if self.backend == H3_DOWN_BACKEND else "32-wide FP32 MMA leaves, BF16 leaves and midpoint tree nodes" ), @@ -242,10 +249,14 @@ def __call__(self, x, weight, *, logical_keys=None, parameter_id="h3.ffn.down.we "forward_executed": False, "input_gradient_executed": False, "weight_gradient_executed": False, + "input_gradient_kernel": None, + "weight_gradient_kernel": None, "qualification": "pending_h3_gpu_evidence", } self.last_execution = trace - return _H3DownFn.apply(x, weight, session, slot, keys, parameter_id, trace) + return _H3DownFn.apply( + x, weight, session, slot, padding_rows, parameter_id, trace, extension + ) def h3_ffn_down_gemm( diff --git a/rl_engine/runtime/registry.py b/rl_engine/runtime/registry.py index 3bb43b404..53a661944 100644 --- a/rl_engine/runtime/registry.py +++ b/rl_engine/runtime/registry.py @@ -215,7 +215,7 @@ def _default_semantic_descriptors() -> tuple[OperatorBackendDescriptor, ...]: supported_dtypes=frozenset({"bfloat16"}), supported_topologies={"world_size": (1,), "tensor_parallel_size": (1,)}, determinism_or_alignment_properties={ - "arithmetic": "fp32_dot_tiles_ordered_fp32_adds_bf16_epilogue", + "arithmetic": "bf16_mma_fp32_accumulation_ordered_fp32_adds_bf16_epilogue", "shape": "14336_to_5376", "weight_gradient_order": "canonical_logical_keys", "gpu_qualified": False, diff --git a/tests/ops/gemm/test_h3_ffn_down.py b/tests/ops/gemm/test_h3_ffn_down.py index 5175693f6..4244ada0a 100644 --- a/tests/ops/gemm/test_h3_ffn_down.py +++ b/tests/ops/gemm/test_h3_ffn_down.py @@ -134,6 +134,8 @@ def test_canonical_chunk_permutation_and_padding_orchestration(cpu_orchestration chunked, traces = training_projection(op, x, w, dy, keys, chunk_size=2) assert all(raw_equal(a, b) for a, b in zip(canonical, chunked, strict=True)) assert sum(trace["weight_gradient_executed"] for trace in traces) == 1 + assert sum(trace["weight_gradient_kernel"] is not None for trace in traces) == 1 + assert all(trace["input_gradient_kernel"] is not None for trace in traces) order = torch.tensor([6, 1, 3, 0, 5, 2, 4]) permuted, _ = training_projection(op, x[order], w, dy[order], keys[order]) assert raw_equal(canonical[2], permuted[2]) @@ -245,6 +247,81 @@ def test_nonzero_padding_gradient_is_rejected(cpu_orchestration): y.backward(torch.ones_like(y)) +def test_backend_is_resolved_once_per_forward_and_retained_for_backward( + cpu_orchestration, monkeypatch +): + resolutions = [] + + def resolve(device, backend): + extension = _CPUOrchestrationOnlyExtension() + resolutions.append(extension) + return extension + + monkeypatch.setattr(down, "_require_backend", resolve) + op = down.H3FFNDownGemmOp() + x = torch.ones(2, 4).bfloat16().requires_grad_(True) + w = torch.ones(3, 4).bfloat16().requires_grad_(True) + with canonical_backward_session() as session: + first = op(x[:1], w, logical_keys=logical_keys(1, "cpu")) + second = op(x[1:], w, logical_keys=logical_keys(1, "cpu") + 1) + assert len(resolutions) == 2 + torch.autograd.backward((first, second), (torch.ones_like(first), torch.ones_like(second))) + session.validate_complete() + assert len(resolutions) == 2 + assert sum(len(ext.weight_reduction_inputs) for ext in resolutions) == 1 + + +def test_unpadded_backward_needs_no_padding_scalar_readback(cpu_orchestration, monkeypatch): + x = torch.ones(2, 4).bfloat16().requires_grad_(True) + w = torch.ones(3, 4).bfloat16().requires_grad_(True) + with canonical_backward_session() as session: + y = down.h3_ffn_down_gemm(x, w, logical_keys=logical_keys(2, "cpu")) + + def unexpected_item(self, *args, **kwargs): + raise AssertionError("unpadded backward must not read a validation scalar") + + with monkeypatch.context() as patch: + patch.setattr(torch.Tensor, "item", unexpected_item) + y.backward(torch.ones_like(y)) + session.validate_complete() + assert x.grad is not None and w.grad is not None + + +@pytest.mark.parametrize("padding_gradient", [1.0, float("nan"), float("inf")]) +def test_padding_validation_uses_forward_snapshot(cpu_orchestration, padding_gradient): + x = torch.ones(2, 4).bfloat16() + w = torch.ones(3, 4).bfloat16().requires_grad_(True) + keys = torch.tensor([[0, 0], [-1, -1]]) + with canonical_backward_session(): + y = down.h3_ffn_down_gemm(x, w, logical_keys=keys) + keys[1] = torch.tensor([1, 1]) + gradient = torch.ones_like(y) + gradient[1] = padding_gradient + with pytest.raises(ValueError, match="zero upstream"): + y.backward(gradient) + assert not cpu_orchestration.weight_reduction_inputs + + +@pytest.mark.parametrize("input_grad,weight_grad", [(True, False), (False, True), (True, True)]) +def test_gradient_kernel_roles_and_runtime_ids( + cpu_orchestration, monkeypatch, input_grad, weight_grad +): + kernel_id = "test.shared_gemm_kernel" + monkeypatch.setattr(cpu_orchestration, "h3_kernel_id", kernel_id, raising=False) + events = [] + monkeypatch.setattr(down, "record_backward", lambda kind, **event: events.append(event)) + x = torch.ones(2, 4).bfloat16().requires_grad_(input_grad) + w = torch.ones(3, 4).bfloat16().requires_grad_(weight_grad) + op = down.H3FFNDownGemmOp() + with canonical_backward_session() as session: + y = op(x, w, logical_keys=logical_keys(2, "cpu")) + y.backward(torch.ones_like(y)) + session.validate_complete() + assert op.last_execution["input_gradient_kernel"] == (kernel_id if input_grad else None) + assert op.last_execution["weight_gradient_kernel"] == (kernel_id if weight_grad else None) + assert len(events) == 1 and events[0]["kernel_id"] == kernel_id + + requires_sm90 = pytest.mark.skipif( not torch.cuda.is_available() or torch.version.hip is not None From 9962635fed4b29ae4d24998be9439d8f44f1718f Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:57:38 +0800 Subject: [PATCH 06/10] Keep H3 fixture capture provenance explicitly unverified Retain capture qualification requirements for self-declared fixture labels and strengthen gradient-call and projection-cardinality checks. Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- docs/operators/h3-ffn-down.md | 15 +++- tests/ops/gemm/test_h3_ffn_down.py | 14 ++++ .../operators/test_h3_down_tooling.py | 83 +++++++++++++++++++ .../operators/validate_h3_ffn_down.py | 11 ++- 4 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index ecb9955d6..5c5c26f8a 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -21,7 +21,10 @@ No new CUDA or Triton device kernel is added. The eager operator accepts BF16 `[M,14336]` or `[B,S,14336]`, flattened row count 1..32768, native `[5376,14336]` weights and no bias on one SM90 CUDA device. It computes `Y=X@W.T`, `dX=dY@W`, and canonical `dW=dY.T@X`. Strided tensors -are normalized by copies. Weight-gradient rows are sorted by logical identity +are normalized by copies. The adapter passes transposed weight/gradient views +using explicit operand strides; storage layout does not change logical K order. +The historical forward/dX/dW matrix includes this path. Weight-gradient rows +are sorted by logical identity and padded to a fixed 32-row extent before a single reduction. The operator rejects invalid shapes, types, devices, TF32, duplicate active @@ -99,6 +102,10 @@ The checkpoint fixture uses `transformer_blocks.0.ff.net.2.weight`, BF16 whole model. Weight SHA256 is `e19d4e1649e4b55b1931842e5cc4bb46892c2e635b9315b2996630a86284e88f`. Activations and upstream gradients are **synthetic**, not captured model data. +The validation CLI preserves fixture labels as declarations and records +`activation_provenance_verified=false`. Even a fixture labeled `captured` keeps +independent capture-provenance verification in its remaining acceptance list; +a file checksum or self-declared capture hash does not establish provenance. The explicit legacy backend `rlkernel.h3_ffn_down.sm90.midtree.v1` failed real-weight forward accuracy at M=129: 23,353 failing elements, maximum error/tolerance @@ -161,6 +168,12 @@ and `tests/runtime/test_layout_compatibility.py`. The three skips require SM90; the new tests exercise CPU orchestration, not GPU arithmetic. Lint and formatting checks passed. No GPU run or performance measurement was made for this fix. +A follow-up to the same review passed **33 tests with 3 SM90 skips** across +the operator and tooling suites after strengthening actual-call assertions +and testing unverified fixture labels. The earlier registry/layout results +were reused; their runtime sources are unchanged. Follow-up lint and formatting +checks passed. No GPU execution was repeated. + ## Reproduction and remaining qualification Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: diff --git a/tests/ops/gemm/test_h3_ffn_down.py b/tests/ops/gemm/test_h3_ffn_down.py index 4244ada0a..a4217496c 100644 --- a/tests/ops/gemm/test_h3_ffn_down.py +++ b/tests/ops/gemm/test_h3_ffn_down.py @@ -100,11 +100,15 @@ class _CPUOrchestrationOnlyExtension: def __init__(self): self.weight_reduction_inputs = [] + self.forward_calls = 0 + self.input_gradient_calls = 0 def det_gemm_fwd_rhs_transposed(self, x, w): + self.forward_calls += 1 return (x.float() @ w.float().t()).bfloat16() def det_gemm_fwd(self, x, w): + self.input_gradient_calls += 1 return (x.float() @ w.float()).bfloat16() def det_gemm_db_transposed(self, x, dy): @@ -320,6 +324,13 @@ def test_gradient_kernel_roles_and_runtime_ids( assert op.last_execution["input_gradient_kernel"] == (kernel_id if input_grad else None) assert op.last_execution["weight_gradient_kernel"] == (kernel_id if weight_grad else None) assert len(events) == 1 and events[0]["kernel_id"] == kernel_id + assert cpu_orchestration.forward_calls == 1 + assert cpu_orchestration.input_gradient_calls == int(input_grad) + assert len(cpu_orchestration.weight_reduction_inputs) == int(weight_grad) + if input_grad: + torch.testing.assert_close(x.grad, torch.full_like(x, 3), atol=0, rtol=0) + if weight_grad: + torch.testing.assert_close(w.grad, torch.full_like(w, 2), atol=0, rtol=0) requires_sm90 = pytest.mark.skipif( @@ -343,11 +354,14 @@ def test_full_h3_accuracy_and_four_judgments_on_gpu(rows): op = down.H3FFNDownGemmOp() actual, _ = training_projection(op, x, w, dy, keys) expected = reference_projection(x, w, dy) + # The projection contract is exactly (forward, input gradient, weight gradient). + assert len(actual) == len(expected) == 3 for index, (left, right) in enumerate(zip(actual, expected, strict=True)): judgment = "forward_accuracy" if index == 0 else "gradient_accuracy" result = comparison(left, right, judgment) assert result["passed"], result chunked, _ = training_projection(op, x, w, dy, keys, chunk_size=31) + assert len(chunked) == 3 for index, (left, right) in enumerate(zip(chunked, actual, strict=True)): judgment = "forward_invariance" if index == 0 else "gradient_invariance" result = comparison(left, right, judgment) diff --git a/tests/validation/operators/test_h3_down_tooling.py b/tests/validation/operators/test_h3_down_tooling.py index e5f4e58bb..f6be7ff58 100644 --- a/tests/validation/operators/test_h3_down_tooling.py +++ b/tests/validation/operators/test_h3_down_tooling.py @@ -9,6 +9,8 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[3] @@ -125,3 +127,84 @@ def test_plot_pipeline_with_explicitly_synthetic_test_data(tmp_path): assert result.returncode == 0, result.stderr assert (tmp_path / "synthetic_test_data.png").read_bytes().startswith(b"\x89PNG") assert "synthetic_test_data" in (tmp_path / "checks.csv").read_text() + + +@pytest.mark.parametrize("activation_kind", ["synthetic", "captured"]) +def test_fixture_label_cannot_clear_capture_qualification(tmp_path, monkeypatch, activation_kind): + import torch + + spec = importlib.util.spec_from_file_location( + "h3_fixture_metadata_under_test", + ROOT / "tools/validation/operators/validate_h3_ffn_down.py", + ) + validator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(validator) + + class FixtureTensor: + """Exercise report control flow without claiming device computation.""" + + dtype = torch.bfloat16 + + def to(self, device): + assert device == "cuda" + return self + + def reshape(self, *shape): + return self + + values = tuple(FixtureTensor() for _ in range(3)) + metadata = { + "checkpoint_revision": validator.H3_CHECKPOINT_REVISION, + "weight_key": "test.down.weight", + "activation_source": "synthetic test fixture claiming a capture", + "activation_kind": activation_kind, + "capture_sha256": "0" * 64, + } + monkeypatch.setattr(validator, "provenance", lambda args: {}) + monkeypatch.setattr(validator, "command", lambda args: "test device metadata") + monkeypatch.setattr(validator, "_require_backend", lambda *args: None) + monkeypatch.setattr(validator, "make_inputs", lambda *args: values) + monkeypatch.setattr(validator, "run_case", lambda *args, **kwargs: {"passed": True}) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_name", lambda: "test device") + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda: (9, 0)) + monkeypatch.setattr(torch.version, "hip", None) + monkeypatch.setattr(torch, "set_float32_matmul_precision", lambda value: None) + monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) + monkeypatch.setattr(torch.backends.cuda.matmul, "allow_bf16_reduced_precision_reduction", False) + monkeypatch.setattr( + torch, + "load", + lambda *args, **kwargs: { + "x": values[0], + "weight": values[1], + "grad_output": values[2], + "metadata": metadata, + }, + ) + fixture_path = tmp_path / "fixture.pt" + fixture_path.write_bytes(b"synthetic control-flow fixture; tensor loading is mocked") + report_path = tmp_path / "report.json" + status = validator.main( + [ + "--rows", + "1", + "--families", + "random", + "--fixture", + str(fixture_path), + "--output", + str(report_path), + ] + ) + report = json.loads(report_path.read_text()) + assert status == 0 and report["status"] == "passed" + assert report["fixture"]["metadata"] == metadata + assert not report["fixture"]["activation_provenance_verified"] + assert not report["acceptance_complete"] + required = ( + "real captured SwiGLU activations" + if activation_kind == "synthetic" + else "independent verification of captured SwiGLU activation provenance" + ) + assert required in report["remaining_acceptance"] diff --git a/tools/validation/operators/validate_h3_ffn_down.py b/tools/validation/operators/validate_h3_ffn_down.py index 8f77acffb..282eed4b3 100644 --- a/tools/validation/operators/validate_h3_ffn_down.py +++ b/tools/validation/operators/validate_h3_ffn_down.py @@ -412,7 +412,12 @@ def save(): raise ValueError("fixture activation_kind must be captured or synthetic") # Metadata must remain serializable even on a later validation failure. json.dumps(metadata, allow_nan=False) - report["fixture"] = {"sha256": sha256(args.fixture), "metadata": metadata} + report["fixture"] = { + "sha256": sha256(args.fixture), + "metadata": metadata, + # A file hash and self-declared label do not attest capture provenance. + "activation_provenance_verified": False, + } values = [fixture[key].to("cuda") for key in ("x", "weight", "grad_output")] if any(value.dtype != torch.bfloat16 for value in values): raise TypeError("fixture tensors must already be quantized BF16") @@ -438,6 +443,10 @@ def save(): report["remaining_acceptance"].append("real checkpoint weights and captured inputs") elif report["fixture"]["metadata"]["activation_kind"] != "captured": report["remaining_acceptance"].append("real captured SwiGLU activations") + else: + report["remaining_acceptance"].append( + "independent verification of captured SwiGLU activation provenance" + ) if not args.build_log and args.backend == H3_LEGACY_DOWN_BACKEND: report["remaining_acceptance"].append("auditable no-fast-math extension build log") exit_code = 0 if passed else 1 From 01745076a2e4ee6d19762e3ca9f3166a7b69aaba Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:53:38 +0800 Subject: [PATCH 07/10] Fix H3 shared GEMM launch and failed accuracy charts Bind the bias-aware shared kernel correctly for forward and both gradients. Keep failed or unavailable accuracy checks visible, add signature and chart regressions, and extend the prepared SM90 canonical-row coverage. Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- .../operators/gemm/plot_h3_down_report.py | 60 ++++++++++---- docs/operators/h3-ffn-down.md | 42 +++++++++- .../triton/gemm/h3_down_fp32_backend.py | 2 + .../triton/gemm/test_h3_down_backend.py | 82 +++++++++++++++++++ tests/benchmarks/test_h3_down_report.py | 68 +++++++++++++++ tests/ops/gemm/test_h3_ffn_down.py | 28 ++++++- 6 files changed, 258 insertions(+), 24 deletions(-) create mode 100644 tests/backends/shared/triton/gemm/test_h3_down_backend.py create mode 100644 tests/benchmarks/test_h3_down_report.py diff --git a/benchmarks/operators/gemm/plot_h3_down_report.py b/benchmarks/operators/gemm/plot_h3_down_report.py index ec0ee9d57..eb75339bd 100644 --- a/benchmarks/operators/gemm/plot_h3_down_report.py +++ b/benchmarks/operators/gemm/plot_h3_down_report.py @@ -6,9 +6,50 @@ import argparse import csv import json +import math from pathlib import Path +def plot_reference_accuracy(axis, cases): + """Keep failed or unavailable accuracy checks visible without inventing errors.""" + counts = [case["rows"] for case in cases] + for tensor in ("y", "dx", "dw"): + values, flagged = [], [] + for case in cases: + checks = [ + row + for row in case["checks"] + if row["tensor"] == tensor and row["transform"] == "fp32_reference" + ] + ratios = [row.get("max_error_over_tolerance") for row in checks] + available = bool(ratios) and all( + value is not None and math.isfinite(value) for value in ratios + ) + values.append(max(ratios) if available else math.nan) + if not available or not all(row["passed"] for row in checks): + flagged.append(case["rows"]) + (line,) = axis.plot(counts, values, marker="o", label=tensor) + if flagged: + # These markers use axes height, not the numerical error scale. + axis.scatter( + flagged, + [1.02] * len(flagged), + transform=axis.get_xaxis_transform(), + clip_on=False, + marker="x", + color=line.get_color(), + label=f"{tensor}: failed / unavailable", + ) + axis.axhline(1, color="red", linestyle="--", label="shared tolerance boundary") + axis.set( + xlabel="Logical rows M", + ylabel="Max error / allowed error", + xscale="log", + title="Accuracy against independent FP32 reference", + ) + axis.legend(fontsize=8) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("report") @@ -117,24 +158,7 @@ def main(): title=family, ) axes[0].legend(fontsize=8) - for tensor in ("y", "dx", "dw"): - values = [ - max( - row.get("max_error_over_tolerance", 0) - for row in case["checks"] - if row["tensor"] == tensor and row["transform"] == "fp32_reference" - ) - for case in subset - ] - axes[1].plot(counts, values, marker="o", label=tensor) - axes[1].axhline(1, color="red", linestyle="--", label="shared tolerance boundary") - axes[1].set( - xlabel="Logical rows M", - ylabel="Max error / allowed error", - xscale="log", - title="Accuracy against independent FP32 reference", - ) - axes[1].legend(fontsize=8) + plot_reference_accuracy(axes[1], subset) fig.savefig(output / f"{family}.png", dpi=180) plt.close(fig) diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index 5c5c26f8a..96db7c682 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -139,18 +139,35 @@ canonical autograd session or full dW invariance matrix. It remains prior art. summary, forward/dX/dW tolerances and worst errors, replay digests, representative timings, rejected native results and original bundle hashes. Full reports, samples, traces and raw originals are preserved privately. This is a selected derivative, -not a lossless report or a new GPU run. Historical published copies remain in Git -history; private archives do not establish public reproducibility. The historical +not a lossless report or a new GPU run. The complete historical report derivatives +are also available from the previously published immutable commit below. These +downloads preserve every case, check, trace and timing sample, including rejected +native results. Raw private originals are unchanged. The historical JSON retains its original FP32-dot trace wording; the corrected BF16 MMA / FP32 accumulation description above applies to that unchanged device computation. +| Historical report download | SHA256 | +|---|---| +| [Summary, real-weight fixture and rejected native results](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json) | `3a44e25a9397373010b75caee1ee321864e960fc83594772497f95bfeab15786` | +| [Random matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-random.json) | `66fcd857d26a57ea03ce0984f8bf20db1785336c61d302dce74dfd6a4cf213fb` | +| [Synthetic SwiGLU-like matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json) | `dcc64cf4360bbb9dca0211319f635cffed1cd91ebc6ddfd24052882a7201ba9d` | +| [Cancellation matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json) | `2088317c29cc9de80540f215f54b7d6d660044a568c1f50f6194449b2c9b4fba` | +| [Cold-cache and compiler-version replays](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json) | `ee532d0a128eed4e978070fcc9fc6358c0cc0b72eafab50266c4645927cb438c` | + +[Original download checksum manifest](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS) +has SHA256 `9d37b26ba4192be72aded2e4278120811dd377246f57b2659a80f489d4997d19`. +These are historical 2026-10-10 executions with portable path substitutions, +not evidence that the current shared GEMM and canonical reducer have been qualified. + - Operator: `rl_engine/ops/gemm/h3_ffn_down.py`. - Triton adapter: `rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py`; actual device computation reuses `_det_gemm_fp32_kernel` in `det_gemm.py`. - Canonical gradients: `rl_engine/ops/autograd/canonical_backward.py`. - Native CUDA comparator: existing SM90 midpoint-tree backend; rejected for the accuracy failure above, never an automatic fallback. -- Tests: `tests/ops/gemm/test_h3_ffn_down.py` and +- Tests: `tests/ops/gemm/test_h3_ffn_down.py`, + `tests/backends/shared/triton/gemm/test_h3_down_backend.py`, + `tests/benchmarks/test_h3_down_report.py` and `tests/validation/operators/test_h3_down_{fixture,tooling}.py`. Historical local checks at `d8ae6cf` passed 58 tests with 13 GPU-dependent skips @@ -174,6 +191,25 @@ and testing unverified fixture labels. The earlier registry/layout results were reused; their runtime sources are unchanged. Follow-up lint and formatting checks passed. No GPU execution was repeated. +## Shared-kernel compatibility update + +The adapter now supplies a null bias pointer and `HAS_BIAS=False` for the +bias-aware shared GEMM signature. CPU regression tests bind the actual forward, +dX and dW calls to the real JIT function signature, including transpose strides +and launch settings. Accuracy charts leave missing/nonfinite ratios unavailable +and mark failed checks explicitly; finite failed ratios remain visible. + +Focused checks on 2026-10-30 with torch 2.6.0+cpu and Triton 3.2.0: +**46 passed, 4 SM90 tests skipped** across the backend, benchmark, operator, fixture +and tooling suites. Lint and formatting passed. CPU binding uses a synthetic +launcher and does not compile or execute a Triton kernel. + +The shared GEMM and canonical reducer changed in the later upstream merge; +the 2026-10-10 H100 results do not qualify these current dependencies. +Current SM90 execution is **not run**. The prepared GPU suite covers full-width +forward/dX/dW at M=1,33,129,4097, chunking, and the large-row ordered/permuted/padded +canonical paths. No current GPU acceptance or performance result is claimed. + ## Reproduction and remaining qualification Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: diff --git a/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py b/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py index a35ffd7ac..c8c71f909 100644 --- a/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py +++ b/rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py @@ -37,6 +37,7 @@ def gemm(self, a, b): a, b, output, + None, m, n, k, @@ -47,6 +48,7 @@ def gemm(self, a, b): BLOCK_N=64, BLOCK_K=32, PROMOTE_INPUTS=False, + HAS_BIAS=False, num_warps=4, num_stages=2, enable_fp_fusion=False, diff --git a/tests/backends/shared/triton/gemm/test_h3_down_backend.py b/tests/backends/shared/triton/gemm/test_h3_down_backend.py new file mode 100644 index 000000000..dab0ed226 --- /dev/null +++ b/tests/backends/shared/triton/gemm/test_h3_down_backend.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""CPU launch binding checks; these do not compile or execute device code.""" + +import inspect +from types import SimpleNamespace + +import pytest +import torch + +from rl_engine.backends.shared.triton.gemm import det_gemm +from rl_engine.backends.shared.triton.gemm.h3_down_fp32_backend import FixedFP32Backend + + +@pytest.mark.parametrize("role", ["forward", "dx", "dw"]) +def test_h3_launch_binds_current_shared_kernel_signature(role): + if not det_gemm._TRITON_AVAILABLE: + pytest.skip("Triton is required to inspect the real kernel signature") + backend = FixedFP32Backend() + signature = inspect.signature(backend.kernel.fn) + launches = [] + + class BindingKernel: + def __getitem__(self, grid): + def launch(*args, **kwargs): + options = { + key: kwargs.pop(key) for key in ("num_warps", "num_stages", "enable_fp_fusion") + } + # Bind the adapter's actual call to the real JIT function, not a + # copied signature that could drift with the implementation. + bound = signature.bind(*args, **kwargs) + bound.apply_defaults() + launches.append((grid, bound.arguments, options)) + return SimpleNamespace( + hash="h3_cpu_binding_fixture", + asm={"cubin": b"synthetic test binary"}, + metadata=SimpleNamespace(num_warps=4), + ) + + return launch + + backend.kernel = BindingKernel() + x = torch.empty((129, 14336), dtype=torch.bfloat16, device="meta") + weight = torch.empty((5376, 14336), dtype=torch.bfloat16, device="meta") + grad = torch.empty((129, 5376), dtype=torch.bfloat16, device="meta") + if role == "forward": + output = backend.det_gemm_fwd_rhs_transposed(x, weight) + dimensions, strides = (129, 5376, 14336), (14336, 1, 1, 14336, 5376, 1) + elif role == "dx": + output = backend.det_gemm_fwd(grad, weight) + dimensions, strides = (129, 14336, 5376), (5376, 1, 14336, 1, 14336, 1) + else: + output = backend.det_gemm_db_transposed(x, grad) + dimensions, strides = (5376, 14336, 129), (1, 5376, 14336, 1, 14336, 1) + assert output.shape == dimensions[:2] + assert output.dtype == torch.bfloat16 + assert len(launches) == 1 + grid, arguments, options = launches[0] + assert grid == tuple((value + 63) // 64 for value in dimensions[:2]) + assert arguments["c_ptr"] is output + assert arguments["bias_ptr"] is None + assert arguments["HAS_BIAS"] is False + assert tuple(arguments[key] for key in ("M", "N", "K")) == dimensions + assert ( + tuple( + arguments[key] + for key in ( + "stride_am", + "stride_ak", + "stride_bk", + "stride_bn", + "stride_cm", + "stride_cn", + ) + ) + == strides + ) + assert tuple(arguments[key] for key in ("BLOCK_M", "BLOCK_N", "BLOCK_K")) == (64, 64, 32) + assert arguments["PROMOTE_INPUTS"] is False + assert arguments["CHUNK_K"] == 0 + assert arguments["EVEN_K"] is arguments["EVEN_N"] is False + assert options == {"num_warps": 4, "num_stages": 2, "enable_fp_fusion": False} diff --git a/tests/benchmarks/test_h3_down_report.py b/tests/benchmarks/test_h3_down_report.py new file mode 100644 index 000000000..8d5e312d8 --- /dev/null +++ b/tests/benchmarks/test_h3_down_report.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Inspect rendered accuracy series, including real nonfinite comparison failures.""" + +import importlib.util +import math +from pathlib import Path + +import matplotlib +import matplotlib.pyplot as plt +import pytest +import torch + +from rl_engine.validation.operators.h3_down_validation import comparison + +matplotlib.use("Agg") + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "h3_report_under_test", ROOT / "benchmarks/operators/gemm/plot_h3_down_report.py" +) +report = importlib.util.module_from_spec(spec) +spec.loader.exec_module(report) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")]) +def test_nonfinite_failure_is_flagged_and_never_plotted_as_zero(value): + failed = comparison(torch.tensor([value]), torch.zeros(1), "forward_accuracy") + assert not failed["passed"] and failed["error"] == "nonfinite tensor" + assert "max_error_over_tolerance" not in failed + checks = [dict(failed, tensor="y", transform="fp32_reference")] + checks += [ + dict(tensor=tensor, transform="fp32_reference", passed=True, max_error_over_tolerance=0.5) + for tensor in ("dx", "dw") + ] + fig, axis = plt.subplots() + try: + report.plot_reference_accuracy(axis, [{"rows": 129, "checks": checks}]) + lines = {line.get_label(): line for line in axis.lines} + assert math.isnan(lines["y"].get_ydata()[0]) + assert lines["dx"].get_ydata()[0] == lines["dw"].get_ydata()[0] == 0.5 + (flag,) = axis.collections + assert flag.get_label() == "y: failed / unavailable" + assert flag.get_offsets()[0, 0] == 129 + assert flag.get_offset_transform() == axis.get_xaxis_transform() + assert flag.get_offsets()[0, 1] > 1 # Outside the numerical plot height. + finally: + plt.close(fig) + + +def test_measured_failure_keeps_error_ratio_and_missing_checks_are_unavailable(): + checks = [ + dict(tensor="y", transform="fp32_reference", passed=False, max_error_over_tolerance=2.5), + dict(tensor="dx", transform="fp32_reference", passed=True, max_error_over_tolerance=0.0), + ] + fig, axis = plt.subplots() + try: + report.plot_reference_accuracy(axis, [{"rows": 33, "checks": checks}]) + lines = {line.get_label(): line for line in axis.lines} + assert lines["y"].get_ydata()[0] == 2.5 + assert lines["dx"].get_ydata()[0] == 0.0 + assert math.isnan(lines["dw"].get_ydata()[0]) + assert {flag.get_label() for flag in axis.collections} == { + "y: failed / unavailable", + "dw: failed / unavailable", + } + finally: + plt.close(fig) diff --git a/tests/ops/gemm/test_h3_ffn_down.py b/tests/ops/gemm/test_h3_ffn_down.py index a4217496c..b8396dadb 100644 --- a/tests/ops/gemm/test_h3_ffn_down.py +++ b/tests/ops/gemm/test_h3_ffn_down.py @@ -343,7 +343,7 @@ def test_gradient_kernel_roles_and_runtime_ids( @pytest.mark.cuda_only @requires_sm90 -@pytest.mark.parametrize("rows", [1, 33, 129]) +@pytest.mark.parametrize("rows", [1, 33, 129, 4097]) def test_full_h3_accuracy_and_four_judgments_on_gpu(rows): torch.backends.cuda.matmul.allow_tf32 = False generator = torch.Generator(device="cuda").manual_seed(420) @@ -352,7 +352,13 @@ def test_full_h3_accuracy_and_four_judgments_on_gpu(rows): dy = (torch.randn(rows, 5376, device="cuda", generator=generator) * 0.1).bfloat16() keys = logical_keys(rows, "cuda") op = down.H3FFNDownGemmOp() - actual, _ = training_projection(op, x, w, dy, keys) + actual, traces = training_projection(op, x, w, dy, keys) + assert len(traces) == 1 + trace = traces[0] + assert trace["actual_backend"] == down.H3_DOWN_BACKEND + assert trace["forward_executed"] and trace["weight_gradient_executed"] + assert trace["input_gradient_kernel"] == trace["weight_gradient_kernel"] == trace["kernel_id"] + assert trace["forward_launch"]["cubin_sha256"] expected = reference_projection(x, w, dy) # The projection contract is exactly (forward, input gradient, weight gradient). assert len(actual) == len(expected) == 3 @@ -360,9 +366,25 @@ def test_full_h3_accuracy_and_four_judgments_on_gpu(rows): judgment = "forward_accuracy" if index == 0 else "gradient_accuracy" result = comparison(left, right, judgment) assert result["passed"], result - chunked, _ = training_projection(op, x, w, dy, keys, chunk_size=31) + chunked, _ = training_projection(op, x, w, dy, keys, chunk_size=31 if rows < 4096 else 2048) assert len(chunked) == 3 for index, (left, right) in enumerate(zip(chunked, actual, strict=True)): judgment = "forward_invariance" if index == 0 else "gradient_invariance" result = comparison(left, right, judgment) assert result["passed"], result + if rows >= 4096: + # Exercise both the CUDA canonical-order reuse and the sorting/filtering + # paths after the shared reducer changed. Keep the logical row set fixed. + order = torch.randperm(rows, device="cuda", generator=generator) + permuted, _ = training_projection(op, x[order], w, dy[order], keys[order]) + assert raw_equal(permuted[0], actual[0][order]) + assert raw_equal(permuted[1], actual[1][order]) + assert raw_equal(permuted[2], actual[2]) + padded_x = torch.cat((x, torch.ones_like(x[:3]))) + padded_dy = torch.cat((dy, torch.zeros_like(dy[:3]))) + padded_keys = torch.cat((keys, torch.full_like(keys[:3], -1))) + padded, _ = training_projection(op, padded_x, w, padded_dy, padded_keys) + assert raw_equal(padded[0][:rows], actual[0]) + assert raw_equal(padded[1][:rows], actual[1]) + assert torch.count_nonzero(padded[1][rows:]) == 0 + assert raw_equal(padded[2], actual[2]) From b5f6dae736f42d3a71214df224ea51d300fcf2f2 Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:02:47 +0800 Subject: [PATCH 08/10] Preserve H3 report backend labels Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- .../operators/gemm/plot_h3_down_report.py | 8 ++- tests/benchmarks/test_h3_down_report.py | 56 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/benchmarks/operators/gemm/plot_h3_down_report.py b/benchmarks/operators/gemm/plot_h3_down_report.py index eb75339bd..19ad18e3b 100644 --- a/benchmarks/operators/gemm/plot_h3_down_report.py +++ b/benchmarks/operators/gemm/plot_h3_down_report.py @@ -59,6 +59,8 @@ def main(): output = Path(args.output_dir) output.mkdir(parents=True, exist_ok=True) cases = report["cases"] + backend = report.get("backend") + backend_label = f"Candidate ({backend})" if backend else "Candidate" # Render earlier native-midtree reports alongside the versioned FP32 candidate. for case in cases: case["benchmarks"] = { @@ -80,7 +82,7 @@ def main(): metadata = report["fixture"]["metadata"] lines += [f"Checkpoint fixture activation kind: **{metadata['activation_kind']}**.", ""] lines += [ - "| Input | M | Checks passed | Total | Candidate fwd ms | torch BF16 fwd ms |", + f"| Input | M | Checks passed | Total | {backend_label} fwd ms | torch BF16 fwd ms |", "|---|---:|---:|---:|---:|---:|", ] for case in cases: @@ -139,9 +141,9 @@ def main(): fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) counts = [case["rows"] for case in subset] for key, label in ( - ("candidate_forward", "Candidate forward"), + ("candidate_forward", f"{backend_label} forward"), ("torch_bf16_forward", "torch BF16 forward"), - ("candidate_forward_backward", "Candidate forward+backward"), + ("candidate_forward_backward", f"{backend_label} forward+backward"), ("torch_bf16_forward_backward", "torch BF16 forward+backward"), ): axes[0].plot( diff --git a/tests/benchmarks/test_h3_down_report.py b/tests/benchmarks/test_h3_down_report.py index 8d5e312d8..494100bac 100644 --- a/tests/benchmarks/test_h3_down_report.py +++ b/tests/benchmarks/test_h3_down_report.py @@ -3,7 +3,9 @@ """Inspect rendered accuracy series, including real nonfinite comparison failures.""" import importlib.util +import json import math +import sys from pathlib import Path import matplotlib @@ -66,3 +68,57 @@ def test_measured_failure_keeps_error_ratio_and_missing_checks_are_unavailable() } finally: plt.close(fig) + + +@pytest.mark.parametrize( + ("backend", "prefix"), + [ + ("rlkernel.h3_ffn_down.triton.fp32_tiles.v1", "candidate"), + ("rlkernel.h3_ffn_down.sm90.midtree.v1", "native"), + (None, "candidate"), + ], +) +def test_report_preserves_backend_identity_in_table_and_plot( + tmp_path, monkeypatch, backend, prefix +): + case = { + "input_family": "random", + "rows": 33, + "checks": [], + "benchmarks": { + f"{prefix}_forward": {"median_ms": 2.0}, + f"{prefix}_forward_backward": {"median_ms": 6.0}, + "torch_bf16_forward": {"median_ms": 1.0}, + "torch_bf16_forward_backward": {"median_ms": 3.0}, + }, + } + payload = {"status": "failed", "cases": [case]} + if backend is not None: + payload["backend"] = backend + source = tmp_path / "report.json" + source.write_text(json.dumps(payload)) + output = tmp_path / "rendered" + monkeypatch.setattr( + sys, "argv", ["plot_h3_down_report.py", str(source), "--output-dir", str(output)] + ) + timing_series = [] + + def inspect_figure(figure, *args, **kwargs): + timing_series.extend( + (line.get_label(), list(line.get_ydata())) for line in figure.axes[0].lines + ) + + monkeypatch.setattr(plt.Figure, "savefig", inspect_figure) + report.main() + + label = f"Candidate ({backend})" if backend else "Candidate" + summary = (output / "summary.md").read_text() + assert f"| {label} fwd ms |" in summary + assert "| random | 33 | 0 | 0 | 2.0000 | 1.0000 |" in summary + assert "Run status: **failed**." in summary + assert timing_series == [ + (f"{label} forward", [2.0]), + ("torch BF16 forward", [1.0]), + (f"{label} forward+backward", [6.0]), + ("torch BF16 forward+backward", [3.0]), + ] From 5db9f1ef97e2a6c6af8b9a32aa66d1d209768461 Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:21:50 +0800 Subject: [PATCH 09/10] Link sanitized H3 historical evidence Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- docs/operators/h3-ffn-down.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index 96db7c682..132930071 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -140,7 +140,7 @@ summary, forward/dX/dW tolerances and worst errors, replay digests, representati timings, rejected native results and original bundle hashes. Full reports, samples, traces and raw originals are preserved privately. This is a selected derivative, not a lossless report or a new GPU run. The complete historical report derivatives -are also available from the previously published immutable commit below. These +are available with device UUIDs removed at the links below. These downloads preserve every case, check, trace and timing sample, including rejected native results. Raw private originals are unchanged. The historical JSON retains its original FP32-dot trace wording; the corrected BF16 MMA / FP32 @@ -148,17 +148,23 @@ accumulation description above applies to that unchanged device computation. | Historical report download | SHA256 | |---|---| -| [Summary, real-weight fixture and rejected native results](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-summary.json) | `3a44e25a9397373010b75caee1ee321864e960fc83594772497f95bfeab15786` | -| [Random matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-random.json) | `66fcd857d26a57ea03ce0984f8bf20db1785336c61d302dce74dfd6a4cf213fb` | -| [Synthetic SwiGLU-like matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-swiglu-like.json) | `dcc64cf4360bbb9dca0211319f635cffed1cd91ebc6ddfd24052882a7201ba9d` | -| [Cancellation matrix](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-cancellation.json) | `2088317c29cc9de80540f215f54b7d6d660044a568c1f50f6194449b2c9b4fba` | -| [Cold-cache and compiler-version replays](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/h100-replays.json) | `ee532d0a128eed4e978070fcc9fc6358c0cc0b72eafab50266c4645927cb438c` | - -[Original download checksum manifest](https://raw.githubusercontent.com/zzf2024/RL-Kernel/d8ae6cf70eb3f18ae6f16c6f0771c5f9a2ea09c6/docs/operators/evidence/h3-down-h100-20261010/SHA256SUMS) -has SHA256 `9d37b26ba4192be72aded2e4278120811dd377246f57b2659a80f489d4997d19`. -These are historical 2026-10-10 executions with portable path substitutions, +| [Summary, real-weight fixture and rejected native results](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/h100-summary.json) | `d6011dd275ffa302abd33814a195d0b31b535da90a6d69988377ec9403bfef8d` | +| [Random matrix](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/h100-random.json) | `66fcd857d26a57ea03ce0984f8bf20db1785336c61d302dce74dfd6a4cf213fb` | +| [Synthetic SwiGLU-like matrix](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/h100-swiglu-like.json) | `dcc64cf4360bbb9dca0211319f635cffed1cd91ebc6ddfd24052882a7201ba9d` | +| [Cancellation matrix](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/h100-cancellation.json) | `2088317c29cc9de80540f215f54b7d6d660044a568c1f50f6194449b2c9b4fba` | +| [Cold-cache and compiler-version replays](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/h100-replays.json) | `31a40fa89e76b80f37b39b12fa2a16a98b3b4bf9d677be5387638ab1ca933cd5` | + +[Download checksum manifest](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/SHA256SUMS) +has SHA256 `d1a268a31cb19458c30ef38f9bf2dc4e340bda2f35ab2525c93cb47d41a3f029`. +These are historical 2026-10-10 executions with portable paths and device UUIDs redacted, not evidence that the current shared GEMM and canonical reducer have been qualified. +[Transformation provenance](https://gist.githubusercontent.com/zzf2024/2225006b9a7c25a05e7df4be423d1331/raw/87293a25635b4d92965cb2c8f1dd1d571c55ad29/README.md), +SHA256 `788a09f028a322ef54eef850fd6ba8394896ad89b8b97cbb52f9b64bf5c7c7a9`, +records the source and derivative hashes. Across the five reports, only four +device UUID occurrences changed; all measurements, failures and original +artifact hashes remain. + - Operator: `rl_engine/ops/gemm/h3_ffn_down.py`. - Triton adapter: `rl_engine/backends/shared/triton/gemm/h3_down_fp32_backend.py`; actual device computation reuses `_det_gemm_fp32_kernel` in `det_gemm.py`. @@ -210,6 +216,12 @@ Current SM90 execution is **not run**. The prepared GPU suite covers full-width forward/dX/dW at M=1,33,129,4097, chunking, and the large-row ordered/permuted/padded canonical paths. No current GPU acceptance or performance result is claimed. +Timing tables and chart legends now identify the recorded backend when available; +reports without that field retain the generic candidate label. The report-label +follow-up passed **12 report/tooling tests** on 2026-10-30; lint and formatting +passed. The earlier compatibility results above are retained as a separate +execution. No GPU execution was repeated. + ## Reproduction and remaining qualification Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: From 92c98f0792f2f26349210f5cff61f572dd15a3c3 Mon Sep 17 00:00:00 2001 From: zzf2024 <188711200+zzf2024@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:42:45 +0800 Subject: [PATCH 10/10] Preserve H3 validation failures when plotting fails Signed-off-by: zzf2024 <188711200+zzf2024@users.noreply.github.com> --- ci/scripts/run_h3_down_h100.sh | 6 +- docs/operators/h3-ffn-down.md | 15 +++- .../operators/test_h3_down_tooling.py | 75 +++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/ci/scripts/run_h3_down_h100.sh b/ci/scripts/run_h3_down_h100.sh index 3dade2bb0..c3b1653b7 100644 --- a/ci/scripts/run_h3_down_h100.sh +++ b/ci/scripts/run_h3_down_h100.sh @@ -71,8 +71,10 @@ if [[ "${H3_COMPARE_VLLM:-0}" == 1 ]]; then args+=(--compare-vllm); fi set +e python tools/validation/operators/validate_h3_ffn_down.py "${args[@]}" 2>&1 | tee "$result_dir/validation.log" validation_status=${PIPESTATUS[0]} +python benchmarks/operators/gemm/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir" 2>&1 | tee "$result_dir/plot.log" +plot_status=$? set -e -python benchmarks/operators/gemm/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir" echo "Evidence directory: $result_dir" if [[ "$gpu_test_status" -ne 0 ]]; then exit "$gpu_test_status"; fi -exit "$validation_status" +if [[ "$validation_status" -ne 0 ]]; then exit "$validation_status"; fi +exit "$plot_status" diff --git a/docs/operators/h3-ffn-down.md b/docs/operators/h3-ffn-down.md index 132930071..46d5b6808 100644 --- a/docs/operators/h3-ffn-down.md +++ b/docs/operators/h3-ffn-down.md @@ -26,6 +26,8 @@ using explicit operand strides; storage layout does not change logical K order. The historical forward/dX/dW matrix includes this path. Weight-gradient rows are sorted by logical identity and padded to a fixed 32-row extent before a single reduction. +The 32768-row limit also applies to the cumulative active rows for one parameter +across a backward session. The operator rejects invalid shapes, types, devices, TF32, duplicate active keys, inconsistent weight identity and nonzero padding gradients. Unsupported @@ -217,13 +219,20 @@ forward/dX/dW at M=1,33,129,4097, chunking, and the large-row ordered/permuted/p canonical paths. No current GPU acceptance or performance result is claimed. Timing tables and chart legends now identify the recorded backend when available; -reports without that field retain the generic candidate label. The report-label -follow-up passed **12 report/tooling tests** on 2026-10-30; lint and formatting -passed. The earlier compatibility results above are retained as a separate +reports without that field retain the generic candidate label. The report and +runbook checks passed **16 report/tooling tests** on 2026-10-30, including +four CPU simulations of failure priority and evidence-directory output. Lint, +formatting and Bash syntax checks passed. The earlier compatibility results above are retained as a separate execution. No GPU execution was repeated. ## Reproduction and remaining qualification +The runbook preserves GPU-test, validation and plotting logs, always prints the +evidence directory after plotting, and returns failures in that priority order. +Its source patch captures tracked edits only; retain any executed untracked +sources separately. Native comparisons also require matching build records; +runtime provenance records the loaded binary hash but does not verify build flags. + Use a dedicated non-MIG H100 80GB and a CUDA 12.4 development environment: ```bash diff --git a/tests/validation/operators/test_h3_down_tooling.py b/tests/validation/operators/test_h3_down_tooling.py index f6be7ff58..ed83b3585 100644 --- a/tests/validation/operators/test_h3_down_tooling.py +++ b/tests/validation/operators/test_h3_down_tooling.py @@ -5,6 +5,7 @@ import hashlib import importlib.util import json +import os import subprocess import sys from pathlib import Path @@ -14,6 +15,80 @@ ROOT = Path(__file__).resolve().parents[3] +@pytest.mark.parametrize( + ("gpu_status", "validation_status", "plot_status", "expected"), + [(17, 23, 29, 17), (0, 23, 29, 23), (0, 0, 29, 29), (0, 0, 0, 0)], + ids=["gpu-failure", "validation-failure", "plot-failure", "success"], +) +def test_runbook_preserves_failure_priority_and_evidence_directory( + tmp_path, monkeypatch, gpu_status, validation_status, plot_status, expected +): + commands = tmp_path / "commands" + commands.mkdir() + # Stub external commands; this exercises shell control flow without a GPU. + stub = ( + f"#!{sys.executable}\n" + + """ +import os +from pathlib import Path +import sys + +command = Path(sys.argv[0]).name +args = sys.argv[1:] +if command == "nvcc": + print("simulated nvcc, release 12.4") +elif command in ("nvidia-smi", "git"): + print("simulated " + command) +elif args == ["-"]: + sys.stdin.read() +elif args[:3] == ["-m", "pip", "freeze"]: + print("simulated-package==0") +elif args[:2] == ["-m", "pytest"]: + if "-k" not in args: + print("simulated GPU test result") + sys.exit(int(os.environ["H3_TEST_GPU_STATUS"])) +elif args[0] == "tools/validation/operators/validate_h3_ffn_down.py": + print("simulated validation result") + status = int(os.environ["H3_TEST_VALIDATION_STATUS"]) + if not status: + Path(os.environ["H3_RESULT_DIR"], "report.json").write_text("{}") + sys.exit(status) +elif args[0] == "benchmarks/operators/gemm/plot_h3_down_report.py": + print("simulated plot result") + sys.exit(int(os.environ["H3_TEST_PLOT_STATUS"])) +else: + raise SystemExit("unexpected stub command: " + repr(args)) +""" + ) + for name in ("python", "nvcc", "nvidia-smi", "git"): + path = commands / name + path.write_text(stub) + path.chmod(0o755) + result_dir = tmp_path / "evidence" + for name, value in { + "PATH": str(commands) + os.pathsep + os.environ["PATH"], + "H3_RESULT_DIR": str(result_dir), + "H3_BUILD_NATIVE": "0", + "H3_TEST_GPU_STATUS": str(gpu_status), + "H3_TEST_VALIDATION_STATUS": str(validation_status), + "H3_TEST_PLOT_STATUS": str(plot_status), + }.items(): + monkeypatch.setenv(name, value) + result = subprocess.run( + ["bash", str(ROOT / "ci/scripts/run_h3_down_h100.sh"), "smoke"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == expected, result.stdout + result.stderr + assert f"Evidence directory: {result_dir}" in result.stdout + assert "simulated GPU test result" in (result_dir / "gpu-tests.txt").read_text() + assert "simulated validation result" in (result_dir / "validation.log").read_text() + assert "simulated plot result" in (result_dir / "plot.log").read_text() + if validation_status: + assert not (result_dir / "report.json").exists() + + def test_gpu_validator_refuses_cpu_and_preserves_error_evidence(tmp_path, monkeypatch): import torch