From 37114afe206575d70b372e534f9826a6f5dd4cae Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:49:56 +0800 Subject: [PATCH] [AMD][Fix] Saturate FP8 KV commits instead of reproducing e4m3fnuz NaN overflow (#42982) --- .../sglang/kernels/ops/kvcache/cache_move.py | 28 ++++++++++- .../kernels/ops/quantization/fp8_kernel.py | 21 ++++++++ python/sglang/srt/mem_cache/memory_pool.py | 11 ++++- python/sglang/test/kernels/prefix_valid.py | 11 ++++- .../mem_cache/test_prefix_valid_fp8_pool.py | 49 ++++++++++++++++++- 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/python/sglang/kernels/ops/kvcache/cache_move.py b/python/sglang/kernels/ops/kvcache/cache_move.py index fc02f8a35e..28c7c5504b 100644 --- a/python/sglang/kernels/ops/kvcache/cache_move.py +++ b/python/sglang/kernels/ops/kvcache/cache_move.py @@ -57,6 +57,25 @@ def set_kv_buffer_prefix_valid_tiled( tl.store(dst_v_row_ptr, v_val, mask=mask_byte) +@triton.jit +def _saturate_to_fp8_range(val, FP8_MAX: tl.constexpr, DTYPE: tl.constexpr): + """Clamp into the FP8 range, leaving NaN untouched. + + Mirrors ``saturate_to_fp8_range`` on the eager side. Comparisons against + NaN are false, so a NaN input falls through both ``tl.where`` arms and + still converts to the FP8 NaN encoding. + + Each arm casts back to ``DTYPE`` because the FP8_MAX literal is FP32 and + would otherwise widen the result. Keeping the value in the source dtype + matters: the final cast to FP8 must start from the same type it did before + this clamp existed, or it can lower to a different conversion with its own + tie-breaking in the FP8 subnormal range. The limit itself is exact in + BF16/FP16/FP32, so the round trip loses nothing. + """ + val = tl.where(val > FP8_MAX, FP8_MAX, val).to(DTYPE) + return tl.where(val < -FP8_MAX, -FP8_MAX, val).to(DTYPE) + + @triton.jit def set_kv_buffer_prefix_valid_tiled_fp8( src_k_ptr, @@ -74,6 +93,7 @@ def set_kv_buffer_prefix_valid_tiled_fp8( block_size, ROW_ELEMS: tl.constexpr, ELEMS_PER_TILE: tl.constexpr, + FP8_MAX: tl.constexpr, K_SCALE_IS_TENSOR: tl.constexpr = False, V_SCALE_IS_TENSOR: tl.constexpr = False, ): @@ -104,7 +124,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8( else: # Eager host-scalar division multiplies by a rounded FP32 reciprocal. k_val = k_val * tl.div_rn(1.0, k_scale) - k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty) + k_val = k_val.to(src_k_ptr.dtype.element_ty) + k_val = _saturate_to_fp8_range(k_val, FP8_MAX, src_k_ptr.dtype.element_ty) + k_val = k_val.to(dst_k_ptr.dtype.element_ty) v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32) if V_SCALE_IS_TENSOR: @@ -112,7 +134,9 @@ def set_kv_buffer_prefix_valid_tiled_fp8( v_val = tl.div_rn(v_val, v_scale) else: v_val = v_val * tl.div_rn(1.0, v_scale) - v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty) + v_val = v_val.to(src_v_ptr.dtype.element_ty) + v_val = _saturate_to_fp8_range(v_val, FP8_MAX, src_v_ptr.dtype.element_ty) + v_val = v_val.to(dst_v_ptr.dtype.element_ty) tl.store(dst_k_row_ptr, k_val, mask=mask_elem) tl.store(dst_v_row_ptr, v_val, mask=mask_elem) diff --git a/python/sglang/kernels/ops/quantization/fp8_kernel.py b/python/sglang/kernels/ops/quantization/fp8_kernel.py index 3706819237..4e48b97e53 100644 --- a/python/sglang/kernels/ops/quantization/fp8_kernel.py +++ b/python/sglang/kernels/ops/quantization/fp8_kernel.py @@ -112,6 +112,27 @@ else: fp8_min = -fp8_max +def saturate_to_fp8_range(value: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + """Clamp to the representable range of ``dtype``, leaving NaN untouched. + + The two FP8 types disagree on overflow. ``float8_e4m3fn`` saturates to its + max finite value, but ``float8_e4m3fnuz`` has no infinity, so its eager cast + returns NaN (``0x80``) for every magnitude at or above 256.0. Clamping first + gives both types the saturating behaviour, which keeps the fused Triton + commit kernel and the eager fallback byte-identical on ROCm and avoids + writing NaN into a committed KV slot. + + Note this is the *representable* limit from ``torch.finfo``, not the + conservative ``fp8_max`` above: clamping at 224.0 would pull values that + eager rounds to 240.0 down a step and lose precision. + """ + limit = torch.finfo(dtype).max + # clamp propagates NaN rather than folding it onto a bound: the CUDA/ROCm + # kernel returns the input early for NaN, and the CPU one leaves it to + # std::min/std::max, which return the NaN operand. + return value.clamp(-limit, limit) + + @triton.jit def _per_token_group_quant_8bit( # Pointers to inputs and output diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 3b66c88c4e..80f50cc56b 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -49,7 +49,11 @@ from sglang.kernels.ops.kvcache.cache_move import ( set_kv_buffer_prefix_valid_tiled_fp8, ) from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache -from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz +from sglang.kernels.ops.quantization.fp8_kernel import ( + fp8_dtype, + is_fp8_fnuz, + saturate_to_fp8_range, +) from sglang.srt.configs.mamba_utils import BaseLinearStateParams from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs @@ -330,6 +334,7 @@ def _set_kv_buffer_prefix_valid_impl_fp8( int(loc_2d.shape[1]), ROW_ELEMS=row_dim, ELEMS_PER_TILE=elems_per_tile, + FP8_MAX=torch.finfo(k_cache.dtype).max, K_SCALE_IS_TENSOR=k_scale_is_tensor, V_SCALE_IS_TENSOR=v_scale_is_tensor, num_warps=num_warps, @@ -3282,6 +3287,10 @@ class MHATokenToKVPool(KVCache): cache_k.div_(k_scale) if v_scale is not None: cache_v.div_(v_scale) + if self.dtype == fp8_dtype: + # Saturate like the fused kernel above; see saturate_to_fp8_range. + cache_k = saturate_to_fp8_range(cache_k, self.dtype) + cache_v = saturate_to_fp8_range(cache_v, self.dtype) cache_k = cache_k.to(self.dtype) cache_v = cache_v.to(self.dtype) diff --git a/python/sglang/test/kernels/prefix_valid.py b/python/sglang/test/kernels/prefix_valid.py index 2f168f9c75..3f0cf2acf2 100644 --- a/python/sglang/test/kernels/prefix_valid.py +++ b/python/sglang/test/kernels/prefix_valid.py @@ -4,7 +4,7 @@ from contextlib import contextmanager import torch -from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype +from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, saturate_to_fp8_range def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"): @@ -45,7 +45,14 @@ def assert_prefix_commit( value.div_(scale) scaled.append(value) target = cache.contiguous().view(torch.uint8).clone() - target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src] + # The commit path saturates out-of-range values rather than reproducing + # the eager cast, which returns NaN on float8_e4m3fnuz. See #42981. + quantized = ( + saturate_to_fp8_range(value, cache.dtype) + if cache.dtype == fp8_dtype + else value + ) + target[dst] = quantized.to(cache.dtype).contiguous().view(torch.uint8)[src] expected.append(target) yield diff --git a/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py b/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py index 8b5b4f5bc6..012066b8b4 100644 --- a/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py +++ b/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py @@ -9,7 +9,11 @@ import torch from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, _resolve_fused_scale from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.kernels.prefix_valid import assert_prefix_commit, make_kv_cache +from sglang.test.kernels.prefix_valid import ( + assert_bytes_equal, + assert_prefix_commit, + make_kv_cache, +) from sglang.test.test_utils import CustomTestCase # Preserve the original GPU lanes; the kernel file retains the other 25 seconds. @@ -191,6 +195,49 @@ class TestPrefixValidFp8Pool(CustomTestCase): layer, loc, lengths, k, v, k_scale, v_scale ) + def test_eager_fallback_saturates_overflow(self): + """Pin the saturation contract without going through the shared helper.""" + # k_scale_float=None keeps _resolve_fused_scale at None, so this takes + # the eager writer rather than the fused kernel. + scale = torch.tensor(0.5, device=DEVICE) + layer = SimpleNamespace( + layer_id=0, + k_scale=scale, + v_scale=scale, + k_scale_float=None, + v_scale_float=None, + ) + # Dividing by 0.5 pushes the first two well past the FP8 range; 1.0 + # lands on 2.0, which is exact in both E4M3 formats. + values = (8192.0, -8192.0, float("nan"), 1.0) + limit = torch.finfo(fp8_dtype).max + + pool = _make_pool(len(values), 6) + before = (pool.k_buffer[0].clone(), pool.v_buffer[0].clone()) + loc = torch.tensor([[1, 4]], device=DEVICE) + lengths = torch.tensor([1], dtype=torch.int32, device=DEVICE) + rows = torch.tensor(values, dtype=torch.bfloat16, device=DEVICE).expand( + 2, 1, -1 + ) + # The eager writer divides in place, so K and V need separate storage. + k, v = rows.clone(), rows.clone() + + pool.set_kv_buffer_prefix_valid(layer, loc, lengths, k, v, scale, scale) + + for label, buffer, original in ( + ("K", pool.k_buffer[0], before[0]), + ("V", pool.v_buffer[0], before[1]), + ): + written = buffer.view(pool.dtype)[1, 0].float() + self.assertEqual(written[0].item(), limit, f"{label}: +overflow saturates") + self.assertEqual(written[1].item(), -limit, f"{label}: -overflow saturates") + self.assertTrue(torch.isnan(written[2]), f"{label}: NaN stays NaN") + self.assertEqual(written[3].item(), 2.0, f"{label}: in-range stays exact") + untouched = [slot for slot in range(buffer.shape[0]) if slot != 1] + assert_bytes_equal( + buffer[untouched], original[untouched], f"{label} uncommitted slots" + ) + class TestDflashFp8PrefixCommit(CustomTestCase): def test_dflash_constructor_initializes_fp8_prefix_scales(self):