[AMD] Fix the QuickReduce bf16 cast failing to build for CDNA (#37132)

Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
YC Yen-Ching Tseng
2026-08-30 20:57:29 -07:00
committed by GitHub
co-authored by Alex Nails
parent 62f86ce470
commit 5972211977
2 changed files with 15 additions and 9 deletions
@@ -498,19 +498,25 @@ struct CodecQ8 : public CodecBase {
} }
}; };
// Keep the scale on the f32 side of the narrowing conversion. With the HIP // Keep the scale on the f32 side of the narrowing conversion. With nothing in
// intrinsic, LLVM can reassociate (bf16_as_f32 * scale) -> fp16 into // between, LLVM reassociates (bf16_as_f32 * scale) -> fp16 into
// fp16(bf16_as_f32) * scale, which clips values above 65504 before the range // fp16(bf16_as_f32) * scale, which clips values above 65504 before the range
// guard is applied. The opaque ISA conversion makes the scaled f32 values // guard is applied.
// explicit inputs and prevents that transform. //
// The barrier is what blocks that: it forces the scaled values into registers
// the optimizer cannot see through, so the multiply has to happen before the
// narrowing. Naming a conversion instruction would do the same, but only where
// that instruction exists -- v_cvt_pk_f16_f32 is not part of the CDNA ISA, so
// spelling it out fails to assemble for gfx942. The narrowing itself is left to
// __float22half2_rn, which every target implements and which matches the
// round-to-nearest conversion the quantized path above uses.
__quickreduce_device_inline__ half2 scaled_bfloat162_to_half2(nv_bfloat162 value, float scale) { __quickreduce_device_inline__ half2 scaled_bfloat162_to_half2(nv_bfloat162 value, float scale) {
float2 scaled = __bfloat1622float2(value); float2 scaled = __bfloat1622float2(value);
scaled.x *= scale; scaled.x *= scale;
scaled.y *= scale; scaled.y *= scale;
int packed; asm volatile("" : "+v"(scaled.x), "+v"(scaled.y));
asm volatile("v_cvt_pk_f16_f32 %0, %1, %2" : "=v"(packed) : "v"(scaled.x), "v"(scaled.y)); return __float22half2_rn(scaled);
return *reinterpret_cast<half2*>(&packed);
} }
// Twoshot All Reduce // Twoshot All Reduce
@@ -114,9 +114,9 @@ buffer_store_dwordx4(int32x4_t data, int32x4_t srsrc, int32_t voffset, int32_t s
__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) { __quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) {
#if defined(__gfx942__) || defined(__gfx950__) #if defined(__gfx942__) || defined(__gfx950__)
if (value) { if (value) {
asm volatile("s_setreg_imm32_b32 0xdc1, 1;" ::: "memory"); asm volatile("s_setreg_imm32_b32 0x5c1, 1;" ::: "memory");
} else { } else {
asm volatile("s_setreg_imm32_b32 0xdc1, 0;" ::: "memory"); asm volatile("s_setreg_imm32_b32 0x5c1, 0;" ::: "memory");
} }
#endif #endif
} }