From cafb37a10dc6994587c98253049efe30a6f3995f Mon Sep 17 00:00:00 2001 From: Jayden Yu <48167665+JiaFengYu@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:47:54 -0400 Subject: [PATCH] [JIT] Fuse FP8 KV-cache quantization into the prefix-valid commit kernel (#40785) Co-authored-by: Jayden Yu Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../sglang/kernels/ops/kvcache/cache_move.py | 61 ++++ python/sglang/srt/mem_cache/memory_pool.py | 156 ++++++++- python/sglang/srt/models/dflash.py | 1 + python/sglang/test/kernels/prefix_valid.py | 60 ++++ .../ops/kvcache/test_prefix_valid_fp8.py | 150 +++++++++ .../mem_cache/test_prefix_valid_fp8_pool.py | 302 ++++++++++++++++++ 6 files changed, 715 insertions(+), 15 deletions(-) create mode 100644 python/sglang/test/kernels/prefix_valid.py create mode 100644 test/registered/kernels/ops/kvcache/test_prefix_valid_fp8.py create mode 100644 test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py diff --git a/python/sglang/kernels/ops/kvcache/cache_move.py b/python/sglang/kernels/ops/kvcache/cache_move.py index cf0e0e4cc2..fc02f8a35e 100644 --- a/python/sglang/kernels/ops/kvcache/cache_move.py +++ b/python/sglang/kernels/ops/kvcache/cache_move.py @@ -57,6 +57,67 @@ def set_kv_buffer_prefix_valid_tiled( tl.store(dst_v_row_ptr, v_val, mask=mask_byte) +@triton.jit +def set_kv_buffer_prefix_valid_tiled_fp8( + src_k_ptr, + src_v_ptr, + dst_k_ptr, + dst_v_ptr, + loc_2d_ptr, + commit_len_ptr, + k_scale, + v_scale, + src_k_row_stride, + src_v_row_stride, + dst_k_row_stride, + dst_v_row_stride, + block_size, + ROW_ELEMS: tl.constexpr, + ELEMS_PER_TILE: tl.constexpr, + K_SCALE_IS_TENSOR: tl.constexpr = False, + V_SCALE_IS_TENSOR: tl.constexpr = False, +): + bid = tl.program_id(0) + row = tl.program_id(1) + tid = tl.program_id(2) + + commit_len = tl.load(commit_len_ptr + bid) + if row >= commit_len: + return + + elem_off = tid * ELEMS_PER_TILE + tl.arange(0, ELEMS_PER_TILE) + mask_elem = elem_off < ROW_ELEMS + + loc = tl.load(loc_2d_ptr + bid * block_size + row) + src_row = bid * block_size + row + + src_k_row_ptr = src_k_ptr + src_row * src_k_row_stride + elem_off + src_v_row_ptr = src_v_ptr + src_row * src_v_row_stride + elem_off + dst_k_row_ptr = dst_k_ptr + loc * dst_k_row_stride + elem_off + dst_v_row_ptr = dst_v_ptr + loc * dst_v_row_stride + elem_off + + # Tensor division rounds the divisor first and uses correctly rounded division. + k_val = tl.load(src_k_row_ptr, mask=mask_elem, other=0).to(tl.float32) + if K_SCALE_IS_TENSOR: + k_scale = k_scale.to(src_k_ptr.dtype.element_ty).to(tl.float32) + k_val = tl.div_rn(k_val, k_scale) + else: + # Eager host-scalar division multiplies by a rounded FP32 reciprocal. + k_val = k_val * tl.div_rn(1.0, k_scale) + k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty) + + v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32) + if V_SCALE_IS_TENSOR: + v_scale = v_scale.to(src_v_ptr.dtype.element_ty).to(tl.float32) + v_val = tl.div_rn(v_val, v_scale) + else: + v_val = v_val * tl.div_rn(1.0, v_scale) + v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty) + + tl.store(dst_k_row_ptr, k_val, mask=mask_elem) + tl.store(dst_v_row_ptr, v_val, mask=mask_elem) + + @triton.jit def copy_all_layer_kv_cache_tiled( data_ptrs, diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 260c4ad9dd..33f6808fa4 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -46,6 +46,7 @@ from sglang.kernels.ops.attention.dsa.quant_k_cache import ( from sglang.kernels.ops.kvcache.cache_move import ( copy_all_layer_kv_cache_func, set_kv_buffer_prefix_valid_tiled, + set_kv_buffer_prefix_valid_tiled_fp8, ) from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz @@ -271,6 +272,98 @@ def _set_kv_buffer_prefix_valid_impl( ) +def _set_kv_buffer_prefix_valid_impl_fp8( + k: torch.Tensor, + v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + k_scale: float, + v_scale: float, + loc_2d: torch.Tensor, + commit_lens: torch.Tensor, + row_dim: int, + k_scale_is_tensor: bool = False, + v_scale_is_tensor: bool = False, +) -> None: + if k.numel() == 0 or loc_2d.numel() == 0 or commit_lens.numel() == 0: + return + + if not k.is_contiguous(): + k = k.contiguous() + if not v.is_contiguous(): + v = v.contiguous() + if not loc_2d.is_contiguous(): + loc_2d = loc_2d.contiguous() + if not commit_lens.is_contiguous(): + commit_lens = commit_lens.contiguous() + + if row_dim <= 0: + return + + if row_dim >= 4096: + elems_per_tile = 256 + num_warps = 8 + elif row_dim >= 2048: + elems_per_tile = 128 + num_warps = 4 + else: + elems_per_tile = 64 + num_warps = 4 + grid = ( + int(loc_2d.shape[0]), + int(loc_2d.shape[1]), + triton.cdiv(row_dim, elems_per_tile), + ) + set_kv_buffer_prefix_valid_tiled_fp8[grid]( + k, + v, + k_cache, + v_cache, + loc_2d, + commit_lens, + k_scale, + v_scale, + int(k.stride(0)), + int(v.stride(0)), + int(k_cache.stride(0)), + int(v_cache.stride(0)), + int(loc_2d.shape[1]), + ROW_ELEMS=row_dim, + ELEMS_PER_TILE=elems_per_tile, + K_SCALE_IS_TENSOR=k_scale_is_tensor, + V_SCALE_IS_TENSOR=v_scale_is_tensor, + num_warps=num_warps, + num_stages=2, + ) + + +def _resolve_fused_scale( + scale, + layer_scale, + layer_scale_float, +) -> Optional[float]: + if isinstance(scale, (float, int)): + return float(scale) + + if ( + isinstance(scale, torch.Tensor) + and scale.ndim == 0 + and scale.device.type == "cpu" + ): + return float(scale.item()) + + if ( + isinstance(scale, torch.Tensor) + and scale.ndim == 0 + and scale.is_cuda + and scale is layer_scale + and isinstance(layer_scale_float, (float, int)) + ): + return float(layer_scale_float) + + return None + + class ReqToTokenPool: """A memory pool that maps a request to its token locations.""" @@ -3104,21 +3197,6 @@ class MHATokenToKVPool(KVCache): f"{tuple(cache_k.shape)=} {tuple(cache_v.shape)=} {tuple(loc_2d.shape)=}." ) - if cache_k.dtype != self.dtype: - if k_scale is not None: - cache_k.div_(k_scale) - if v_scale is not None: - cache_v.div_(v_scale) - cache_k = cache_k.to(self.dtype) - cache_v = cache_v.to(self.dtype) - - if self.store_dtype != self.dtype: - cache_k = cache_k.contiguous().view(self.store_dtype) - cache_v = cache_v.contiguous().view(self.store_dtype) - else: - cache_k = cache_k.contiguous() - cache_v = cache_v.contiguous() - if loc_2d.device != self.k_buffer[0].device: loc_2d = loc_2d.to(device=self.k_buffer[0].device, non_blocking=True) if commit_lens.device != self.k_buffer[0].device: @@ -3156,6 +3234,54 @@ class MHATokenToKVPool(KVCache): f"head_dim={self.head_dim} v_head_dim={self.v_head_dim}." ) + fused_k_scale = _resolve_fused_scale( + k_scale, + getattr(layer, "k_scale", None), + getattr(layer, "k_scale_float", None), + ) + fused_v_scale = _resolve_fused_scale( + v_scale, + getattr(layer, "v_scale", None), + getattr(layer, "v_scale_float", None), + ) + # fuse quantization and writing into the same kernel + if ( + cache_k.dtype != self.dtype + and self.dtype == fp8_dtype + and fused_k_scale is not None + and fused_v_scale is not None + ): + _set_kv_buffer_prefix_valid_impl_fp8( + cache_k, + cache_v, + self.k_buffer[layer_id - self.start_layer].view(self.dtype), + self.v_buffer[layer_id - self.start_layer].view(self.dtype), + fused_k_scale, + fused_v_scale, + loc_2d, + commit_lens, + row_dim=self.row_dim, + k_scale_is_tensor=isinstance(k_scale, torch.Tensor) and k_scale.is_cuda, + v_scale_is_tensor=isinstance(v_scale, torch.Tensor) and v_scale.is_cuda, + ) + return + + # fallback: eager quantization + if cache_k.dtype != self.dtype: + if k_scale is not None: + cache_k.div_(k_scale) + if v_scale is not None: + cache_v.div_(v_scale) + cache_k = cache_k.to(self.dtype) + cache_v = cache_v.to(self.dtype) + + if self.store_dtype != self.dtype: + cache_k = cache_k.contiguous().view(self.store_dtype) + cache_v = cache_v.contiguous().view(self.store_dtype) + else: + cache_k = cache_k.contiguous() + cache_v = cache_v.contiguous() + _set_kv_buffer_prefix_valid_impl( cache_k, cache_v, diff --git a/python/sglang/srt/models/dflash.py b/python/sglang/srt/models/dflash.py index 05b6f1f8d8..ddd22d508e 100644 --- a/python/sglang/srt/models/dflash.py +++ b/python/sglang/srt/models/dflash.py @@ -316,6 +316,7 @@ class DFlashAttention(nn.Module): layer_id=layer_id, sliding_window_size=self.sliding_window_size, attn_type=self.attn_type, + quant_config=quant_config, ) def forward_prepare_npu(self, positions, hidden_states): diff --git a/python/sglang/test/kernels/prefix_valid.py b/python/sglang/test/kernels/prefix_valid.py new file mode 100644 index 0000000000..2f168f9c75 --- /dev/null +++ b/python/sglang/test/kernels/prefix_valid.py @@ -0,0 +1,60 @@ +"""Shared eager oracle and byte assertions for prefix-valid KV-cache tests.""" + +from contextlib import contextmanager + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype + + +def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"): + store_dtype = torch.uint8 if dtype == fp8_dtype else dtype + return tuple( + torch.full( + (total_slots, 1, row_dim), fill, dtype=store_dtype, device=device + ).view(dtype) + for fill in (0x5A, 0xA5) + ) + + +def assert_bytes_equal(actual, expected, label): + # Integer comparison preserves NaN encodings and the sign of zero. + torch.testing.assert_close( + actual.contiguous().view(torch.uint8), + expected.contiguous().view(torch.uint8), + rtol=0, + atol=0, + msg=lambda message: f"{label}: {message}", + ) + + +@contextmanager +def assert_prefix_commit( + k, v, k_cache, v_cache, loc, lengths, k_scale, v_scale, *, inputs_mutated=False +): + """Check the whole cache (including untouched slots) against eager division.""" + rows = torch.arange(loc.shape[1], device=loc.device) + valid = (rows[None, :] < lengths[:, None]).reshape(-1) + src = torch.nonzero(valid, as_tuple=False).flatten() + dst = loc.reshape(-1)[src].long() + before = (k.clone(), v.clone()) + scaled, expected = [], [] + for source, cache, scale in zip(before, (k_cache, v_cache), (k_scale, v_scale)): + value = source.clone() + if scale is not None: + value.div_(scale) + scaled.append(value) + target = cache.contiguous().view(torch.uint8).clone() + target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src] + expected.append(target) + + yield + + for label, cache, reference in zip( + ("K cache", "V cache"), (k_cache, v_cache), expected + ): + assert_bytes_equal(cache, reference, label) + for label, source, reference in zip( + ("K input", "V input"), (k, v), scaled if inputs_mutated else before + ): + assert_bytes_equal(source, reference, label) diff --git a/test/registered/kernels/ops/kvcache/test_prefix_valid_fp8.py b/test/registered/kernels/ops/kvcache/test_prefix_valid_fp8.py new file mode 100644 index 0000000000..2beb9ab587 --- /dev/null +++ b/test/registered/kernels/ops/kvcache/test_prefix_valid_fp8.py @@ -0,0 +1,150 @@ +"""Numerical and masking regressions for fused prefix-valid FP8 commits.""" + +import unittest + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype +from sglang.srt.mem_cache.memory_pool import _set_kv_buffer_prefix_valid_impl_fp8 +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.prefix_valid import assert_prefix_commit, make_kv_cache +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-large") +# backend-specific: ROCm's E4M3FNUZ pointer inference and conversion differ from CUDA. +register_amd_ci(est_time=25, stage="jit-kernel-unit", runner_config="amd") + +DEVICE = "cuda" +SOURCE_DTYPES = (torch.bfloat16, torch.float16, torch.float32) + + +class TestPrefixValidFp8Kernel(CustomTestCase): + def test_matches_eager_bytes_and_preserves_uncommitted_slots(self): + """Cover tile branches, noncontiguous inputs, tails and mixed commit lengths.""" + batch_size, block_size = 3, 5 + loc = torch.arange(1, 16, device=DEVICE).view(batch_size, block_size) + lengths = torch.tensor([0, 3, 5], dtype=torch.int32, device=DEVICE) + for dtype in SOURCE_DTYPES: + for row_dim, noncontiguous in ((63, True), (2048, False), (4097, False)): + with self.subTest(dtype=dtype, row_dim=row_dim): + torch.manual_seed(1234 + row_dim) + if noncontiguous: + source = torch.randn( + (15, 2, row_dim), dtype=dtype, device=DEVICE + ) + k, v = source[:, :1], source[:, 1:] + self.assertFalse(k.is_contiguous()) + self.assertFalse(v.is_contiguous()) + else: + k = torch.randn((15, 1, row_dim), dtype=dtype, device=DEVICE) + v = torch.randn_like(k) + k_cache, v_cache = make_kv_cache(20, row_dim) + with assert_prefix_commit( + k, v, k_cache, v_cache, loc, lengths, 0.375, 1.75 + ): + _set_kv_buffer_prefix_valid_impl_fp8( + k, v, k_cache, v_cache, 0.375, 1.75, loc, lengths, row_dim + ) + + def test_matches_eager_at_fp8_boundaries(self): + """Representable limits, NaNs and signed zero follow eager FP8 casting.""" + fp8_max = torch.finfo(fp8_dtype).max + values = [ + -fp8_max, + -fp8_max + 1.0, + -0.75 * fp8_max, + -1.0625, + -1.0, + -0.0, + 0.0, + 1.0, + 1.0625, + 0.75 * fp8_max, + fp8_max - 1.0, + fp8_max, + float("nan"), + ] + loc = torch.tensor([[3]], device=DEVICE) + lengths = torch.tensor([1], dtype=torch.int32, device=DEVICE) + for dtype in SOURCE_DTYPES: + with self.subTest(dtype=dtype): + k = torch.tensor(values, dtype=dtype, device=DEVICE).view(1, 1, -1) + v = k.flip(-1).contiguous() + k_cache, v_cache = make_kv_cache(8, len(values)) + with assert_prefix_commit( + k, v, k_cache, v_cache, loc, lengths, 1.0, 1.0 + ): + _set_kv_buffer_prefix_valid_impl_fp8( + k, v, k_cache, v_cache, 1.0, 1.0, loc, lengths, len(values) + ) + + def test_scale_division_at_rounding_boundaries(self): + """Host and GPU scales must retain their distinct eager rounding semantics.""" + loc = torch.tensor([[0]], device=DEVICE) + lengths = torch.tensor([1], dtype=torch.int32, device=DEVICE) + scales = (0.1, 0.3, 0.7, 1.3, 0.625, 1.375, 1e-4, 1e4, 2.0**-140) + for dtype in SOURCE_DTYPES: + for value in scales: + scale = torch.nn.Parameter( + torch.tensor(value, dtype=torch.float32, device=DEVICE), + requires_grad=False, + ) + if dtype == torch.float32: + fp8 = ( + torch.arange(127, dtype=torch.uint8, device=DEVICE) + .view(fp8_dtype) + .float() + ) + mid = (fp8[:-1] + fp8[1:]) * 0.5 * scale + values = torch.stack( + ( + torch.nextafter(mid, torch.full_like(mid, -float("inf"))), + mid, + torch.nextafter(mid, torch.full_like(mid, float("inf"))), + ) + ).flatten() + values = torch.cat((values, -values)) + else: + # Every BF16/FP16 bit pattern, including subnormals and NaNs. + values = ( + torch.arange(65536, dtype=torch.int32, device=DEVICE) + .to(torch.int16) + .view(dtype) + ) + k = values.view(1, 1, -1) + v = k.flip(-1) + k_cache, v_cache = make_kv_cache(1, k.shape[-1]) + scalar = scale.item() + for kind, original_scale in ( + ("gpu", scale), + ("python", scalar), + ("cpu", scale.cpu()), + ): + with self.subTest(dtype=dtype, scale=value, scale_kind=kind): + with assert_prefix_commit( + k, + v, + k_cache, + v_cache, + loc, + lengths, + original_scale, + original_scale, + ): + _set_kv_buffer_prefix_valid_impl_fp8( + k, + v, + k_cache, + v_cache, + scalar, + scalar, + loc, + lengths, + k.shape[-1], + k_scale_is_tensor=kind == "gpu", + v_scale_is_tensor=kind == "gpu", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py b/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py new file mode 100644 index 0000000000..8b5b4f5bc6 --- /dev/null +++ b/test/registered/unit/mem_cache/test_prefix_valid_fp8_pool.py @@ -0,0 +1,302 @@ +"""Pool dispatch and real DFLASH initialization for FP8 prefix commits.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, _resolve_fused_scale +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.prefix_valid import assert_prefix_commit, make_kv_cache +from sglang.test.test_utils import CustomTestCase + +# Preserve the original GPU lanes; the kernel file retains the other 25 seconds. +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") +# backend-specific: loaded FP8 scales are doubled for ROCm's E4M3FNUZ format. +register_amd_ci(est_time=15, stage="jit-kernel-unit", runner_config="amd") + +DEVICE = "cuda" + + +def _make_pool(row_dim, total_slots, dtype=fp8_dtype): + # Isolate dispatch from the allocator; real DFLASH initialization is tested below. + pool = MHATokenToKVPool.__new__(MHATokenToKVPool) + pool.dtype = dtype + pool.store_dtype = torch.uint8 if dtype == fp8_dtype else dtype + pool.start_layer = 0 + pool.row_dim = pool.v_row_dim = pool.head_dim = pool.v_head_dim = row_dim + k, v = make_kv_cache(total_slots, row_dim, dtype) + pool.k_buffer, pool.v_buffer = ( + [k.view(pool.store_dtype)], + [v.view(pool.store_dtype)], + ) + return pool + + +def _make_layer(k_scale, v_scale): + scales = [ + torch.nn.Parameter( + torch.tensor(value, dtype=torch.float32, device=DEVICE), requires_grad=False + ) + for value in (k_scale, v_scale) + ] + return SimpleNamespace( + layer_id=0, + k_scale=scales[0], + v_scale=scales[1], + k_scale_float=scales[0].item(), + v_scale_float=scales[1].item(), + ) + + +def _check_pool_commit( + pool, k, v, loc, lengths, k_scale, v_scale, *, inputs_mutated=False +): + return assert_prefix_commit( + k, + v, + pool.k_buffer[0].view(pool.dtype), + pool.v_buffer[0].view(pool.dtype), + loc, + lengths, + k_scale, + v_scale, + inputs_mutated=inputs_mutated, + ) + + +class TestPrefixValidFp8Pool(CustomTestCase): + def setUp(self): + super().setUp() + torch.manual_seed(0) + + def test_pool_dispatch_uses_gpu_parameter_float_shadows(self): + """GPU scale Parameters must fuse without synchronizing through item().""" + pool, layer = _make_pool(65, 12), _make_layer(0.625, 1.375) + loc = torch.tensor([[2, 7, 9]], dtype=torch.int32, device=DEVICE) + lengths = torch.tensor([2], dtype=torch.int64, device=DEVICE) + k = torch.randn((3, 1, 65), dtype=torch.bfloat16, device=DEVICE) + v = torch.randn_like(k) + with _check_pool_commit(pool, k, v, loc, lengths, layer.k_scale, layer.v_scale): + with patch.object( + torch.Tensor, + "item", + side_effect=AssertionError("dispatch called item()"), + ): + pool.set_kv_buffer_prefix_valid( + layer, loc, lengths, k, v, layer.k_scale, layer.v_scale + ) + + def test_scale_kind_preserves_eager_division(self): + """Preserve the BF16 0.1 counterexample and independent K/V scale kinds.""" + loc = torch.tensor([[2, -1, -1], [5, 1, -1]], device=DEVICE) + lengths = torch.tensor([1, 2], dtype=torch.int32, device=DEVICE) + kinds = ( + ("gpu", "gpu"), + ("gpu", "python"), + ("python", "gpu"), + ("cpu", "gpu"), + ("gpu", "cpu"), + ) + for dtype in (torch.bfloat16, torch.float16, torch.float32): + for k_kind, v_kind in kinds: + with self.subTest(dtype=dtype, k_kind=k_kind, v_kind=v_kind): + pool, layer = _make_pool(65, 8), _make_layer(0.1, 0.3) + k_scale = { + "gpu": layer.k_scale, + "python": layer.k_scale_float, + "cpu": layer.k_scale.cpu(), + }[k_kind] + v_scale = { + "gpu": layer.v_scale, + "python": layer.v_scale_float, + "cpu": layer.v_scale.cpu(), + }[v_kind] + k = ( + torch.linspace(-2, 2, 390, device=DEVICE) + .to(dtype) + .view(6, 1, 65) + ) + k[:, :, 0] = 0.0966796875 + v = k.flip(-1) + with _check_pool_commit(pool, k, v, loc, lengths, k_scale, v_scale): + pool.set_kv_buffer_prefix_valid( + layer, loc, lengths, k, v, k_scale, v_scale + ) + + def test_host_scale_rounding_through_pool(self): + """Host reciprocal rounding must not move FP32 inputs onto FP8 ties.""" + loc = torch.tensor([[3, -1]], device=DEVICE) + lengths = torch.tensor([1], dtype=torch.int32, device=DEVICE) + for scale in (10000.0, torch.tensor(10000.0)): + with self.subTest(scale_type=type(scale).__name__): + pool, layer = _make_pool(65, 8), SimpleNamespace(layer_id=0) + k = torch.full( + (2, 1, 65), 29.296873092651367, dtype=torch.float32, device=DEVICE + ) + v = -k + with _check_pool_commit(pool, k, v, loc, lengths, scale, scale): + pool.set_kv_buffer_prefix_valid( + layer, loc, lengths, k, v, scale, scale + ) + + def test_eager_fallbacks(self): + """Unresolved/missing scales and non-FP8 storage retain the old writer.""" + unresolved = SimpleNamespace( + layer_id=0, + k_scale=torch.tensor(0.625, device=DEVICE), + v_scale=torch.tensor(1.375, device=DEVICE), + k_scale_float=None, + v_scale_float=None, + ) + missing = SimpleNamespace( + layer_id=0, + k_scale=None, + v_scale=1.375, + k_scale_float=None, + v_scale_float=1.375, + ) + cases = ( + ( + "unresolved_gpu_scales", + fp8_dtype, + unresolved, + unresolved.k_scale, + unresolved.v_scale, + True, + ), + ("missing_k_scale", fp8_dtype, missing, None, missing.v_scale, True), + ( + "non_fp8_cache", + torch.bfloat16, + SimpleNamespace(layer_id=0), + None, + None, + False, + ), + ) + loc = torch.tensor([[2, 7, 9]], device=DEVICE) + lengths = torch.tensor([2], dtype=torch.int32, device=DEVICE) + for name, dtype, layer, k_scale, v_scale, mutated in cases: + with self.subTest(case=name): + pool = _make_pool(65, 12, dtype) + k = torch.randn((3, 1, 65), dtype=torch.bfloat16, device=DEVICE) + v = torch.randn_like(k) + with _check_pool_commit( + pool, k, v, loc, lengths, k_scale, v_scale, inputs_mutated=mutated + ): + pool.set_kv_buffer_prefix_valid( + layer, loc, lengths, k, v, k_scale, v_scale + ) + + +class TestDflashFp8PrefixCommit(CustomTestCase): + def test_dflash_constructor_initializes_fp8_prefix_scales(self): + """A real FP8 draft must initialize scales and reach the fused writer.""" + from transformers import LlamaConfig + + from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8KVCacheMethod + from sglang.srt.models.dflash import DFlashAttention + from sglang.srt.runtime_context import get_context, get_parallel + + config = LlamaConfig( + hidden_size=128, + intermediate_size=256, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=64, + max_position_embeddings=128, + ) + with ( + get_context().override_server_args(tp_size=1), + get_parallel().override(tp_rank=0), + ): + with torch.device(DEVICE): + plain = DFlashAttention(config, layer_id=0).attn + self.assertIsNone(plain.quant_method) + self.assertIsNone(plain.k_scale) + self.assertIsNone(plain.v_scale) + + for loaded_scales in (None, (0.625, 1.375)): + with self.subTest(loaded_scales=loaded_scales): + with torch.device(DEVICE): + layer = DFlashAttention( + config, layer_id=0, quant_config=Fp8Config() + ).attn + self.assertIsInstance(layer.quant_method, Fp8KVCacheMethod) + for scale in (layer.k_scale, layer.v_scale): + self.assertIsInstance(scale, torch.nn.Parameter) + self.assertTrue(scale.is_cuda) + self.assertEqual(scale.dtype, torch.float32) + self.assertEqual(scale.ndim, 0) + self.assertEqual(scale.item(), -1.0) + if loaded_scales is not None: + with torch.no_grad(): + layer.k_scale.fill_(loaded_scales[0]) + layer.v_scale.fill_(loaded_scales[1]) + layer.quant_method.process_weights_after_loading(layer) + factor = 2 if loaded_scales and is_fp8_fnuz() else 1 + expected_scales = tuple( + value * factor for value in (loaded_scales or (1.0, 1.0)) + ) + self.assertEqual( + (layer.k_scale_float, layer.v_scale_float), expected_scales + ) + self.assertEqual( + (layer.k_scale.item(), layer.v_scale.item()), expected_scales + ) + + pool = _make_pool(64, 8) + loc = torch.tensor([[2, 5, -1]], device=DEVICE) + lengths = torch.tensor([2], dtype=torch.int32, device=DEVICE) + k = ( + torch.linspace(-2, 2, 192, device=DEVICE) + .to(torch.bfloat16) + .view(3, 1, 64) + ) + v = k.flip(-1) + with _check_pool_commit( + pool, k, v, loc, lengths, layer.k_scale, layer.v_scale + ): + # Run real GPU stores; reject only the unintended fallback. + with patch( + "sglang.srt.mem_cache.memory_pool._set_kv_buffer_prefix_valid_impl", + side_effect=AssertionError( + "FP8 draft took the eager fallback" + ), + ): + pool.set_kv_buffer_prefix_valid( + layer, loc, lengths, k, v, layer.k_scale, layer.v_scale + ) + + +class TestResolveFusedScale(CustomTestCase): + def test_supported_and_fallback_scale_forms(self): + """Only synchronization-free scalar forms may enter the fused kernel.""" + layer_scale = torch.tensor(0.5, dtype=torch.float32, device=DEVICE) + unrelated_scale = torch.tensor(0.75, dtype=torch.float32, device=DEVICE) + vector_scale = layer_scale.view(1) + cases = ( + (0.25, None, None, 0.25), + (2, None, None, 2.0), + (torch.tensor(0.5), None, None, 0.5), + (layer_scale, layer_scale, 0.5, 0.5), + (unrelated_scale, layer_scale, 0.5, None), + (None, None, 0.5, None), + (vector_scale, vector_scale, 0.5, None), + (torch.tensor([0.5]), None, None, None), + ) + for scale, owner, shadow, expected in cases: + with self.subTest( + scale_type=type(scale).__name__, + has_owner=owner is not None, + shadow=shadow, + ): + self.assertEqual(_resolve_fused_scale(scale, owner, shadow), expected) + + +if __name__ == "__main__": + unittest.main()