[AMD][Fix] Saturate FP8 KV commits instead of reproducing e4m3fnuz NaN overflow (#42982)

This commit is contained in:
Michael
2026-10-10 09:49:56 +08:00
committed by GitHub
parent 77ba54f7df
commit 37114afe20
5 changed files with 114 additions and 6 deletions
@@ -57,6 +57,25 @@ def set_kv_buffer_prefix_valid_tiled(
tl.store(dst_v_row_ptr, v_val, mask=mask_byte)
@triton.jit
def _saturate_to_fp8_range(val, FP8_MAX: tl.constexpr, DTYPE: tl.constexpr):
"""Clamp into the FP8 range, leaving NaN untouched.
Mirrors ``saturate_to_fp8_range`` on the eager side. Comparisons against
NaN are false, so a NaN input falls through both ``tl.where`` arms and
still converts to the FP8 NaN encoding.
Each arm casts back to ``DTYPE`` because the FP8_MAX literal is FP32 and
would otherwise widen the result. Keeping the value in the source dtype
matters: the final cast to FP8 must start from the same type it did before
this clamp existed, or it can lower to a different conversion with its own
tie-breaking in the FP8 subnormal range. The limit itself is exact in
BF16/FP16/FP32, so the round trip loses nothing.
"""
val = tl.where(val > FP8_MAX, FP8_MAX, val).to(DTYPE)
return tl.where(val < -FP8_MAX, -FP8_MAX, val).to(DTYPE)
@triton.jit
def set_kv_buffer_prefix_valid_tiled_fp8(
src_k_ptr,
@@ -74,6 +93,7 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
block_size,
ROW_ELEMS: tl.constexpr,
ELEMS_PER_TILE: tl.constexpr,
FP8_MAX: tl.constexpr,
K_SCALE_IS_TENSOR: tl.constexpr = False,
V_SCALE_IS_TENSOR: tl.constexpr = False,
):
@@ -104,7 +124,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
else:
# Eager host-scalar division multiplies by a rounded FP32 reciprocal.
k_val = k_val * tl.div_rn(1.0, k_scale)
k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty)
k_val = k_val.to(src_k_ptr.dtype.element_ty)
k_val = _saturate_to_fp8_range(k_val, FP8_MAX, src_k_ptr.dtype.element_ty)
k_val = k_val.to(dst_k_ptr.dtype.element_ty)
v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32)
if V_SCALE_IS_TENSOR:
@@ -112,7 +134,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
v_val = tl.div_rn(v_val, v_scale)
else:
v_val = v_val * tl.div_rn(1.0, v_scale)
v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty)
v_val = v_val.to(src_v_ptr.dtype.element_ty)
v_val = _saturate_to_fp8_range(v_val, FP8_MAX, src_v_ptr.dtype.element_ty)
v_val = v_val.to(dst_v_ptr.dtype.element_ty)
tl.store(dst_k_row_ptr, k_val, mask=mask_elem)
tl.store(dst_v_row_ptr, v_val, mask=mask_elem)
@@ -112,6 +112,27 @@ else:
fp8_min = -fp8_max
def saturate_to_fp8_range(value: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
"""Clamp to the representable range of ``dtype``, leaving NaN untouched.
The two FP8 types disagree on overflow. ``float8_e4m3fn`` saturates to its
max finite value, but ``float8_e4m3fnuz`` has no infinity, so its eager cast
returns NaN (``0x80``) for every magnitude at or above 256.0. Clamping first
gives both types the saturating behaviour, which keeps the fused Triton
commit kernel and the eager fallback byte-identical on ROCm and avoids
writing NaN into a committed KV slot.
Note this is the *representable* limit from ``torch.finfo``, not the
conservative ``fp8_max`` above: clamping at 224.0 would pull values that
eager rounds to 240.0 down a step and lose precision.
"""
limit = torch.finfo(dtype).max
# clamp propagates NaN rather than folding it onto a bound: the CUDA/ROCm
# kernel returns the input early for NaN, and the CPU one leaves it to
# std::min/std::max, which return the NaN operand.
return value.clamp(-limit, limit)
@triton.jit
def _per_token_group_quant_8bit(
# Pointers to inputs and output
+10 -1
View File
@@ -49,7 +49,11 @@ from sglang.kernels.ops.kvcache.cache_move import (
set_kv_buffer_prefix_valid_tiled_fp8,
)
from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz
from sglang.kernels.ops.quantization.fp8_kernel import (
fp8_dtype,
is_fp8_fnuz,
saturate_to_fp8_range,
)
from sglang.srt.configs.mamba_utils import BaseLinearStateParams
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
@@ -330,6 +334,7 @@ def _set_kv_buffer_prefix_valid_impl_fp8(
int(loc_2d.shape[1]),
ROW_ELEMS=row_dim,
ELEMS_PER_TILE=elems_per_tile,
FP8_MAX=torch.finfo(k_cache.dtype).max,
K_SCALE_IS_TENSOR=k_scale_is_tensor,
V_SCALE_IS_TENSOR=v_scale_is_tensor,
num_warps=num_warps,
@@ -3282,6 +3287,10 @@ class MHATokenToKVPool(KVCache):
cache_k.div_(k_scale)
if v_scale is not None:
cache_v.div_(v_scale)
if self.dtype == fp8_dtype:
# Saturate like the fused kernel above; see saturate_to_fp8_range.
cache_k = saturate_to_fp8_range(cache_k, self.dtype)
cache_v = saturate_to_fp8_range(cache_v, self.dtype)
cache_k = cache_k.to(self.dtype)
cache_v = cache_v.to(self.dtype)
+9 -2
View File
@@ -4,7 +4,7 @@ from contextlib import contextmanager
import torch
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, saturate_to_fp8_range
def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"):
@@ -45,7 +45,14 @@ def assert_prefix_commit(
value.div_(scale)
scaled.append(value)
target = cache.contiguous().view(torch.uint8).clone()
target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src]
# The commit path saturates out-of-range values rather than reproducing
# the eager cast, which returns NaN on float8_e4m3fnuz. See #42981.
quantized = (
saturate_to_fp8_range(value, cache.dtype)
if cache.dtype == fp8_dtype
else value
)
target[dst] = quantized.to(cache.dtype).contiguous().view(torch.uint8)[src]
expected.append(target)
yield