diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index 7e944b16902..60b390bdcf4 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -897,7 +897,7 @@ def gen_mad_mix_bf16( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(result));' ) else: - L.append(f' uint16_t h = util::f32_to_bf16(result);') + L.append(f' uint16_t h = util::f32_to_bf16_rne(result);') if result == 'lo': L.append( f' ::rocjitsu::amdgpu::write_vop3_true16_dst({d}, wf, lane, 0u, h);' diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index 6e3a16e548a..ee7c7ab894e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -394,7 +394,7 @@ def test_mad_mixlo_bf16_uses_true16_low_write(): assert 'vdst.write_lane(wf, lane, (prev & 0xFFFF0000u)' not in cpp -def test_gfx1250_bf16_mad_mix_variants_use_bf16_helper(): +def test_gfx1250_bf16_mad_mix_variants_use_rne_helper(): cpp_f32 = gen_mad_mix_bf16( ['vdst'], ['src0', 'src1', 'src2'], @@ -411,7 +411,18 @@ def test_gfx1250_bf16_mad_mix_variants_use_bf16_helper(): opsel_exprs=('inst_.opsel', 'inst_.opsel_hi'), use_cdna5_helpers=True, ) + cpp_hi = gen_mad_mix_bf16( + ['vdst'], + ['src0', 'src1', 'src2'], + result='hi', + op_sel_hi_2_expr='inst_.pad_14', + opsel_exprs=('inst_.opsel', 'inst_.opsel_hi'), + use_cdna5_helpers=True, + ) assert 'read_fma_mix_bf16_source_f32(src0, wf, lane' in cpp_f32 assert 'std::bit_cast(result)' in cpp_f32 - assert 'util::f32_to_bf16(result)' in cpp_lo + assert 'util::f32_to_bf16_rne(result)' in cpp_lo + assert 'util::f32_to_bf16_rne(result)' in cpp_hi + assert 'util::f32_to_bf16(result)' not in cpp_lo + assert 'util::f32_to_bf16(result)' not in cpp_hi diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index 08be7bcd440..9439fce6d13 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -1949,7 +1949,7 @@ void VFmaMixloBf16Vop3p::execute_impl(amdgpu::Wavefront &wf) { float result = std::fma(a, b, c); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_bf16(result); + uint16_t h = util::f32_to_bf16_rne(result); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } } @@ -1995,7 +1995,7 @@ RJ_NOINLINE void VFmaMixloBf16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf float result = std::fma(a, b, c); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_bf16(result); + uint16_t h = util::f32_to_bf16_rne(result); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } dpp_write_mask_scope_.restore(); @@ -2031,7 +2031,7 @@ void VFmaMixhiBf16Vop3p::execute_impl(amdgpu::Wavefront &wf) { float result = std::fma(a, b, c); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_bf16(result); + uint16_t h = util::f32_to_bf16_rne(result); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } } @@ -2077,7 +2077,7 @@ RJ_NOINLINE void VFmaMixhiBf16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf float result = std::fma(a, b, c); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_bf16(result); + uint16_t h = util::f32_to_bf16_rne(result); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } dpp_write_mask_scope_.restore(); diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index cbfa23f69ac..b3b62c1fa53 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -1530,6 +1530,81 @@ TEST(Gfx1250ExecutionTest, PkFmaF32SimdMatchesScalarWithPartialExec) { } } +TEST(Gfx1250ExecutionTest, FmaMixBf16ResultsUseRoundToNearestEven) { + struct TestCase { + uint32_t input; + uint16_t expected; + }; + constexpr std::array test_cases{ + TestCase{0x3f800000u, 0x3f80u}, // Exact. + TestCase{0x3f807fffu, 0x3f80u}, // Just below halfway. + TestCase{0x3f808000u, 0x3f80u}, // Halfway, retained LSB even. + TestCase{0x3f808001u, 0x3f81u}, // Just above halfway. + TestCase{0x3f818000u, 0x3f82u}, // Halfway, retained LSB odd. + TestCase{0xbf818000u, 0xbf82u}, // Negative halfway, retained LSB odd. + TestCase{0x00018000u, 0x0002u}, // Subnormal halfway, retained LSB odd. + TestCase{0x7f7f8000u, 0x7f80u}, // Finite value rounds to infinity. + }; + constexpr uint32_t kDstSeed = 0xa5a55a5au; + constexpr uint32_t kIdentityQuadPerm = 0xe4u; + + for (const uint16_t opcode : {cdna5::kVFmaMixloBf16Vop3p, cdna5::kVFmaMixhiBf16Vop3p}) { + for (const bool use_dpp : {false, true}) { + SCOPED_TRACE(opcode == cdna5::kVFmaMixloBf16Vop3p ? "mixlo" : "mixhi"); + SCOPED_TRACE(use_dpp ? "dpp" : "ordinary"); + + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr instruction; + if (use_dpp) { + cdna5::Vop3pVopDpp16MachineInst raw{}; + raw.vdst = 3; + raw.op = opcode; + raw.encoding = 0xccu; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 257; + raw.src2 = 258; + raw.opsel_hi = 2; + raw.vsrc0 = 0; + raw.dpp_ctrl = kIdentityQuadPerm; + raw.fi = 1; + raw.bound_ctrl = 0; + raw.bank_mask = 0xfu; + raw.row_mask = 0xfu; + static_assert(sizeof(raw) == 3 * sizeof(uint32_t)); + instruction.reset(decode_valid(*decoder, reinterpret_cast(&raw))); + } else { + const auto words = cdna5::build_vop3p( + opcode, {.vdst = 3, .src0 = 256, .src1 = 257, .src2 = 258, .opsel_hi = 2}); + instruction.reset(decode_valid(*decoder, words.data())); + } + ASSERT_NE(instruction, nullptr); + + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec((uint64_t{1} << test_cases.size()) - 1); + const uint32_t vgpr_base = wf->vgpr_alloc().base; + for (uint32_t lane = 0; lane < test_cases.size(); ++lane) { + cu->write_vgpr(vgpr_base, lane, test_cases[lane].input); + cu->write_vgpr(vgpr_base + 1, lane, 0xcafe3f80u); // BF16 1.0 in low half. + cu->write_vgpr(vgpr_base + 2, lane, 0u); + cu->write_vgpr(vgpr_base + 3, lane, kDstSeed); + } + + cu->execute_instruction(instruction.get(), *wf); + for (uint32_t lane = 0; lane < test_cases.size(); ++lane) { + const uint32_t expected = + opcode == cdna5::kVFmaMixloBf16Vop3p + ? (kDstSeed & 0xffff0000u) | test_cases[lane].expected + : (static_cast(test_cases[lane].expected) << 16) | (kDstSeed & 0xffffu); + EXPECT_EQ(cu->read_vgpr(vgpr_base + 3, lane), expected) << "lane " << lane; + } + } + } +} + TEST(Gfx1250DecodeTest, Vop3pRejectsLiteral64SelectorInEverySourcePosition) { auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); ASSERT_NE(decoder, nullptr);