[AMD] Fix the QuickReduce bf16 cast failing to build for CDNA (#37132)
Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
co-authored by
Alex Nails
parent
62f86ce470
commit
5972211977
@@ -498,19 +498,25 @@ struct CodecQ8 : public CodecBase {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Keep the scale on the f32 side of the narrowing conversion. With the HIP
|
// Keep the scale on the f32 side of the narrowing conversion. With nothing in
|
||||||
// intrinsic, LLVM can reassociate (bf16_as_f32 * scale) -> fp16 into
|
// between, LLVM reassociates (bf16_as_f32 * scale) -> fp16 into
|
||||||
// fp16(bf16_as_f32) * scale, which clips values above 65504 before the range
|
// fp16(bf16_as_f32) * scale, which clips values above 65504 before the range
|
||||||
// guard is applied. The opaque ISA conversion makes the scaled f32 values
|
// guard is applied.
|
||||||
// explicit inputs and prevents that transform.
|
//
|
||||||
|
// The barrier is what blocks that: it forces the scaled values into registers
|
||||||
|
// the optimizer cannot see through, so the multiply has to happen before the
|
||||||
|
// narrowing. Naming a conversion instruction would do the same, but only where
|
||||||
|
// that instruction exists -- v_cvt_pk_f16_f32 is not part of the CDNA ISA, so
|
||||||
|
// spelling it out fails to assemble for gfx942. The narrowing itself is left to
|
||||||
|
// __float22half2_rn, which every target implements and which matches the
|
||||||
|
// round-to-nearest conversion the quantized path above uses.
|
||||||
__quickreduce_device_inline__ half2 scaled_bfloat162_to_half2(nv_bfloat162 value, float scale) {
|
__quickreduce_device_inline__ half2 scaled_bfloat162_to_half2(nv_bfloat162 value, float scale) {
|
||||||
float2 scaled = __bfloat1622float2(value);
|
float2 scaled = __bfloat1622float2(value);
|
||||||
scaled.x *= scale;
|
scaled.x *= scale;
|
||||||
scaled.y *= scale;
|
scaled.y *= scale;
|
||||||
|
|
||||||
int packed;
|
asm volatile("" : "+v"(scaled.x), "+v"(scaled.y));
|
||||||
asm volatile("v_cvt_pk_f16_f32 %0, %1, %2" : "=v"(packed) : "v"(scaled.x), "v"(scaled.y));
|
return __float22half2_rn(scaled);
|
||||||
return *reinterpret_cast<half2*>(&packed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Twoshot All Reduce
|
// Twoshot All Reduce
|
||||||
|
|||||||
@@ -114,9 +114,9 @@ buffer_store_dwordx4(int32x4_t data, int32x4_t srsrc, int32_t voffset, int32_t s
|
|||||||
__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) {
|
__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) {
|
||||||
#if defined(__gfx942__) || defined(__gfx950__)
|
#if defined(__gfx942__) || defined(__gfx950__)
|
||||||
if (value) {
|
if (value) {
|
||||||
asm volatile("s_setreg_imm32_b32 0xdc1, 1;" ::: "memory");
|
asm volatile("s_setreg_imm32_b32 0x5c1, 1;" ::: "memory");
|
||||||
} else {
|
} else {
|
||||||
asm volatile("s_setreg_imm32_b32 0xdc1, 0;" ::: "memory");
|
asm volatile("s_setreg_imm32_b32 0x5c1, 0;" ::: "memory");
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user