[AMD][Fix] Saturate FP8 KV commits instead of reproducing e4m3fnuz NaN overflow (#42982)
This commit is contained in:
@@ -57,6 +57,25 @@ def set_kv_buffer_prefix_valid_tiled(
|
||||
tl.store(dst_v_row_ptr, v_val, mask=mask_byte)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _saturate_to_fp8_range(val, FP8_MAX: tl.constexpr, DTYPE: tl.constexpr):
|
||||
"""Clamp into the FP8 range, leaving NaN untouched.
|
||||
|
||||
Mirrors ``saturate_to_fp8_range`` on the eager side. Comparisons against
|
||||
NaN are false, so a NaN input falls through both ``tl.where`` arms and
|
||||
still converts to the FP8 NaN encoding.
|
||||
|
||||
Each arm casts back to ``DTYPE`` because the FP8_MAX literal is FP32 and
|
||||
would otherwise widen the result. Keeping the value in the source dtype
|
||||
matters: the final cast to FP8 must start from the same type it did before
|
||||
this clamp existed, or it can lower to a different conversion with its own
|
||||
tie-breaking in the FP8 subnormal range. The limit itself is exact in
|
||||
BF16/FP16/FP32, so the round trip loses nothing.
|
||||
"""
|
||||
val = tl.where(val > FP8_MAX, FP8_MAX, val).to(DTYPE)
|
||||
return tl.where(val < -FP8_MAX, -FP8_MAX, val).to(DTYPE)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def set_kv_buffer_prefix_valid_tiled_fp8(
|
||||
src_k_ptr,
|
||||
@@ -74,6 +93,7 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
|
||||
block_size,
|
||||
ROW_ELEMS: tl.constexpr,
|
||||
ELEMS_PER_TILE: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
K_SCALE_IS_TENSOR: tl.constexpr = False,
|
||||
V_SCALE_IS_TENSOR: tl.constexpr = False,
|
||||
):
|
||||
@@ -104,7 +124,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
|
||||
else:
|
||||
# Eager host-scalar division multiplies by a rounded FP32 reciprocal.
|
||||
k_val = k_val * tl.div_rn(1.0, k_scale)
|
||||
k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty)
|
||||
k_val = k_val.to(src_k_ptr.dtype.element_ty)
|
||||
k_val = _saturate_to_fp8_range(k_val, FP8_MAX, src_k_ptr.dtype.element_ty)
|
||||
k_val = k_val.to(dst_k_ptr.dtype.element_ty)
|
||||
|
||||
v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32)
|
||||
if V_SCALE_IS_TENSOR:
|
||||
@@ -112,7 +134,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8(
|
||||
v_val = tl.div_rn(v_val, v_scale)
|
||||
else:
|
||||
v_val = v_val * tl.div_rn(1.0, v_scale)
|
||||
v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty)
|
||||
v_val = v_val.to(src_v_ptr.dtype.element_ty)
|
||||
v_val = _saturate_to_fp8_range(v_val, FP8_MAX, src_v_ptr.dtype.element_ty)
|
||||
v_val = v_val.to(dst_v_ptr.dtype.element_ty)
|
||||
|
||||
tl.store(dst_k_row_ptr, k_val, mask=mask_elem)
|
||||
tl.store(dst_v_row_ptr, v_val, mask=mask_elem)
|
||||
|
||||
@@ -112,6 +112,27 @@ else:
|
||||
fp8_min = -fp8_max
|
||||
|
||||
|
||||
def saturate_to_fp8_range(value: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""Clamp to the representable range of ``dtype``, leaving NaN untouched.
|
||||
|
||||
The two FP8 types disagree on overflow. ``float8_e4m3fn`` saturates to its
|
||||
max finite value, but ``float8_e4m3fnuz`` has no infinity, so its eager cast
|
||||
returns NaN (``0x80``) for every magnitude at or above 256.0. Clamping first
|
||||
gives both types the saturating behaviour, which keeps the fused Triton
|
||||
commit kernel and the eager fallback byte-identical on ROCm and avoids
|
||||
writing NaN into a committed KV slot.
|
||||
|
||||
Note this is the *representable* limit from ``torch.finfo``, not the
|
||||
conservative ``fp8_max`` above: clamping at 224.0 would pull values that
|
||||
eager rounds to 240.0 down a step and lose precision.
|
||||
"""
|
||||
limit = torch.finfo(dtype).max
|
||||
# clamp propagates NaN rather than folding it onto a bound: the CUDA/ROCm
|
||||
# kernel returns the input early for NaN, and the CPU one leaves it to
|
||||
# std::min/std::max, which return the NaN operand.
|
||||
return value.clamp(-limit, limit)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _per_token_group_quant_8bit(
|
||||
# Pointers to inputs and output
|
||||
|
||||
@@ -49,7 +49,11 @@ from sglang.kernels.ops.kvcache.cache_move import (
|
||||
set_kv_buffer_prefix_valid_tiled_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
fp8_dtype,
|
||||
is_fp8_fnuz,
|
||||
saturate_to_fp8_range,
|
||||
)
|
||||
from sglang.srt.configs.mamba_utils import BaseLinearStateParams
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
@@ -330,6 +334,7 @@ def _set_kv_buffer_prefix_valid_impl_fp8(
|
||||
int(loc_2d.shape[1]),
|
||||
ROW_ELEMS=row_dim,
|
||||
ELEMS_PER_TILE=elems_per_tile,
|
||||
FP8_MAX=torch.finfo(k_cache.dtype).max,
|
||||
K_SCALE_IS_TENSOR=k_scale_is_tensor,
|
||||
V_SCALE_IS_TENSOR=v_scale_is_tensor,
|
||||
num_warps=num_warps,
|
||||
@@ -3282,6 +3287,10 @@ class MHATokenToKVPool(KVCache):
|
||||
cache_k.div_(k_scale)
|
||||
if v_scale is not None:
|
||||
cache_v.div_(v_scale)
|
||||
if self.dtype == fp8_dtype:
|
||||
# Saturate like the fused kernel above; see saturate_to_fp8_range.
|
||||
cache_k = saturate_to_fp8_range(cache_k, self.dtype)
|
||||
cache_v = saturate_to_fp8_range(cache_v, self.dtype)
|
||||
cache_k = cache_k.to(self.dtype)
|
||||
cache_v = cache_v.to(self.dtype)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, saturate_to_fp8_range
|
||||
|
||||
|
||||
def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"):
|
||||
@@ -45,7 +45,14 @@ def assert_prefix_commit(
|
||||
value.div_(scale)
|
||||
scaled.append(value)
|
||||
target = cache.contiguous().view(torch.uint8).clone()
|
||||
target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src]
|
||||
# The commit path saturates out-of-range values rather than reproducing
|
||||
# the eager cast, which returns NaN on float8_e4m3fnuz. See #42981.
|
||||
quantized = (
|
||||
saturate_to_fp8_range(value, cache.dtype)
|
||||
if cache.dtype == fp8_dtype
|
||||
else value
|
||||
)
|
||||
target[dst] = quantized.to(cache.dtype).contiguous().view(torch.uint8)[src]
|
||||
expected.append(target)
|
||||
|
||||
yield
|
||||
|
||||
Reference in New Issue
Block a user