[AMD] Fix the QuickReduce bf16 cast failing to build for CDNA (#37132)

Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
YC Yen-Ching Tseng
2026-08-30 20:57:29 -07:00
committed by GitHub
co-authored by Alex Nails
parent 62f86ce470
commit 5972211977
2 changed files with 15 additions and 9 deletions
@@ -498,19 +498,25 @@ struct CodecQ8 : public CodecBase {
}
};
// Keep the scale on the f32 side of the narrowing conversion. With the HIP
// intrinsic, LLVM can reassociate (bf16_as_f32 * scale) -> fp16 into
// Keep the scale on the f32 side of the narrowing conversion. With nothing in
// between, LLVM reassociates (bf16_as_f32 * scale) -> fp16 into
// fp16(bf16_as_f32) * scale, which clips values above 65504 before the range
// guard is applied. The opaque ISA conversion makes the scaled f32 values
// explicit inputs and prevents that transform.
// guard is applied.
//
// The barrier is what blocks that: it forces the scaled values into registers
// the optimizer cannot see through, so the multiply has to happen before the
// narrowing. Naming a conversion instruction would do the same, but only where
// that instruction exists -- v_cvt_pk_f16_f32 is not part of the CDNA ISA, so
// spelling it out fails to assemble for gfx942. The narrowing itself is left to
// __float22half2_rn, which every target implements and which matches the
// round-to-nearest conversion the quantized path above uses.
__quickreduce_device_inline__ half2 scaled_bfloat162_to_half2(nv_bfloat162 value, float scale) {
float2 scaled = __bfloat1622float2(value);
scaled.x *= scale;
scaled.y *= scale;
int packed;
asm volatile("v_cvt_pk_f16_f32 %0, %1, %2" : "=v"(packed) : "v"(scaled.x), "v"(scaled.y));
return *reinterpret_cast<half2*>(&packed);
asm volatile("" : "+v"(scaled.x), "+v"(scaled.y));
return __float22half2_rn(scaled);
}
// Twoshot All Reduce
@@ -114,9 +114,9 @@ buffer_store_dwordx4(int32x4_t data, int32x4_t srsrc, int32_t voffset, int32_t s
__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) {
#if defined(__gfx942__) || defined(__gfx950__)
if (value) {
asm volatile("s_setreg_imm32_b32 0xdc1, 1;" ::: "memory");
asm volatile("s_setreg_imm32_b32 0x5c1, 1;" ::: "memory");
} else {
asm volatile("s_setreg_imm32_b32 0xdc1, 0;" ::: "memory");
asm volatile("s_setreg_imm32_b32 0x5c1, 0;" ::: "memory");
}
#endif
}