[JIT] Fuse FP8 KV-cache quantization into the prefix-valid commit kernel (#40785)

Co-authored-by: Jayden Yu <jaydenyu@fb.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Jayden Yu
2026-10-05 14:47:54 +08:00
committed by GitHub
co-authored by Jayden Yu Xiaoyu Zhang
parent f218798006
commit cafb37a10d
6 changed files with 715 additions and 15 deletions
@@ -57,6 +57,67 @@ def set_kv_buffer_prefix_valid_tiled(
tl.store(dst_v_row_ptr, v_val, mask=mask_byte)
@triton.jit
def set_kv_buffer_prefix_valid_tiled_fp8(
src_k_ptr,
src_v_ptr,
dst_k_ptr,
dst_v_ptr,
loc_2d_ptr,
commit_len_ptr,
k_scale,
v_scale,
src_k_row_stride,
src_v_row_stride,
dst_k_row_stride,
dst_v_row_stride,
block_size,
ROW_ELEMS: tl.constexpr,
ELEMS_PER_TILE: tl.constexpr,
K_SCALE_IS_TENSOR: tl.constexpr = False,
V_SCALE_IS_TENSOR: tl.constexpr = False,
):
bid = tl.program_id(0)
row = tl.program_id(1)
tid = tl.program_id(2)
commit_len = tl.load(commit_len_ptr + bid)
if row >= commit_len:
return
elem_off = tid * ELEMS_PER_TILE + tl.arange(0, ELEMS_PER_TILE)
mask_elem = elem_off < ROW_ELEMS
loc = tl.load(loc_2d_ptr + bid * block_size + row)
src_row = bid * block_size + row
src_k_row_ptr = src_k_ptr + src_row * src_k_row_stride + elem_off
src_v_row_ptr = src_v_ptr + src_row * src_v_row_stride + elem_off
dst_k_row_ptr = dst_k_ptr + loc * dst_k_row_stride + elem_off
dst_v_row_ptr = dst_v_ptr + loc * dst_v_row_stride + elem_off
# Tensor division rounds the divisor first and uses correctly rounded division.
k_val = tl.load(src_k_row_ptr, mask=mask_elem, other=0).to(tl.float32)
if K_SCALE_IS_TENSOR:
k_scale = k_scale.to(src_k_ptr.dtype.element_ty).to(tl.float32)
k_val = tl.div_rn(k_val, k_scale)
else:
# Eager host-scalar division multiplies by a rounded FP32 reciprocal.
k_val = k_val * tl.div_rn(1.0, k_scale)
k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty)
v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32)
if V_SCALE_IS_TENSOR:
v_scale = v_scale.to(src_v_ptr.dtype.element_ty).to(tl.float32)
v_val = tl.div_rn(v_val, v_scale)
else:
v_val = v_val * tl.div_rn(1.0, v_scale)
v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty)
tl.store(dst_k_row_ptr, k_val, mask=mask_elem)
tl.store(dst_v_row_ptr, v_val, mask=mask_elem)
@triton.jit
def copy_all_layer_kv_cache_tiled(
data_ptrs,
+141 -15
View File
@@ -46,6 +46,7 @@ from sglang.kernels.ops.attention.dsa.quant_k_cache import (
from sglang.kernels.ops.kvcache.cache_move import (
copy_all_layer_kv_cache_func,
set_kv_buffer_prefix_valid_tiled,
set_kv_buffer_prefix_valid_tiled_fp8,
)
from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz
@@ -271,6 +272,98 @@ def _set_kv_buffer_prefix_valid_impl(
)
def _set_kv_buffer_prefix_valid_impl_fp8(
k: torch.Tensor,
v: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
k_scale: float,
v_scale: float,
loc_2d: torch.Tensor,
commit_lens: torch.Tensor,
row_dim: int,
k_scale_is_tensor: bool = False,
v_scale_is_tensor: bool = False,
) -> None:
if k.numel() == 0 or loc_2d.numel() == 0 or commit_lens.numel() == 0:
return
if not k.is_contiguous():
k = k.contiguous()
if not v.is_contiguous():
v = v.contiguous()
if not loc_2d.is_contiguous():
loc_2d = loc_2d.contiguous()
if not commit_lens.is_contiguous():
commit_lens = commit_lens.contiguous()
if row_dim <= 0:
return
if row_dim >= 4096:
elems_per_tile = 256
num_warps = 8
elif row_dim >= 2048:
elems_per_tile = 128
num_warps = 4
else:
elems_per_tile = 64
num_warps = 4
grid = (
int(loc_2d.shape[0]),
int(loc_2d.shape[1]),
triton.cdiv(row_dim, elems_per_tile),
)
set_kv_buffer_prefix_valid_tiled_fp8[grid](
k,
v,
k_cache,
v_cache,
loc_2d,
commit_lens,
k_scale,
v_scale,
int(k.stride(0)),
int(v.stride(0)),
int(k_cache.stride(0)),
int(v_cache.stride(0)),
int(loc_2d.shape[1]),
ROW_ELEMS=row_dim,
ELEMS_PER_TILE=elems_per_tile,
K_SCALE_IS_TENSOR=k_scale_is_tensor,
V_SCALE_IS_TENSOR=v_scale_is_tensor,
num_warps=num_warps,
num_stages=2,
)
def _resolve_fused_scale(
scale,
layer_scale,
layer_scale_float,
) -> Optional[float]:
if isinstance(scale, (float, int)):
return float(scale)
if (
isinstance(scale, torch.Tensor)
and scale.ndim == 0
and scale.device.type == "cpu"
):
return float(scale.item())
if (
isinstance(scale, torch.Tensor)
and scale.ndim == 0
and scale.is_cuda
and scale is layer_scale
and isinstance(layer_scale_float, (float, int))
):
return float(layer_scale_float)
return None
class ReqToTokenPool:
"""A memory pool that maps a request to its token locations."""
@@ -3104,21 +3197,6 @@ class MHATokenToKVPool(KVCache):
f"{tuple(cache_k.shape)=} {tuple(cache_v.shape)=} {tuple(loc_2d.shape)=}."
)
if cache_k.dtype != self.dtype:
if k_scale is not None:
cache_k.div_(k_scale)
if v_scale is not None:
cache_v.div_(v_scale)
cache_k = cache_k.to(self.dtype)
cache_v = cache_v.to(self.dtype)
if self.store_dtype != self.dtype:
cache_k = cache_k.contiguous().view(self.store_dtype)
cache_v = cache_v.contiguous().view(self.store_dtype)
else:
cache_k = cache_k.contiguous()
cache_v = cache_v.contiguous()
if loc_2d.device != self.k_buffer[0].device:
loc_2d = loc_2d.to(device=self.k_buffer[0].device, non_blocking=True)
if commit_lens.device != self.k_buffer[0].device:
@@ -3156,6 +3234,54 @@ class MHATokenToKVPool(KVCache):
f"head_dim={self.head_dim} v_head_dim={self.v_head_dim}."
)
fused_k_scale = _resolve_fused_scale(
k_scale,
getattr(layer, "k_scale", None),
getattr(layer, "k_scale_float", None),
)
fused_v_scale = _resolve_fused_scale(
v_scale,
getattr(layer, "v_scale", None),
getattr(layer, "v_scale_float", None),
)
# fuse quantization and writing into the same kernel
if (
cache_k.dtype != self.dtype
and self.dtype == fp8_dtype
and fused_k_scale is not None
and fused_v_scale is not None
):
_set_kv_buffer_prefix_valid_impl_fp8(
cache_k,
cache_v,
self.k_buffer[layer_id - self.start_layer].view(self.dtype),
self.v_buffer[layer_id - self.start_layer].view(self.dtype),
fused_k_scale,
fused_v_scale,
loc_2d,
commit_lens,
row_dim=self.row_dim,
k_scale_is_tensor=isinstance(k_scale, torch.Tensor) and k_scale.is_cuda,
v_scale_is_tensor=isinstance(v_scale, torch.Tensor) and v_scale.is_cuda,
)
return
# fallback: eager quantization
if cache_k.dtype != self.dtype:
if k_scale is not None:
cache_k.div_(k_scale)
if v_scale is not None:
cache_v.div_(v_scale)
cache_k = cache_k.to(self.dtype)
cache_v = cache_v.to(self.dtype)
if self.store_dtype != self.dtype:
cache_k = cache_k.contiguous().view(self.store_dtype)
cache_v = cache_v.contiguous().view(self.store_dtype)
else:
cache_k = cache_k.contiguous()
cache_v = cache_v.contiguous()
_set_kv_buffer_prefix_valid_impl(
cache_k,
cache_v,
+1
View File
@@ -316,6 +316,7 @@ class DFlashAttention(nn.Module):
layer_id=layer_id,
sliding_window_size=self.sliding_window_size,
attn_type=self.attn_type,
quant_config=quant_config,
)
def forward_prepare_npu(self, positions, hidden_states):
@@ -0,0 +1,60 @@
"""Shared eager oracle and byte assertions for prefix-valid KV-cache tests."""
from contextlib import contextmanager
import torch
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"):
store_dtype = torch.uint8 if dtype == fp8_dtype else dtype
return tuple(
torch.full(
(total_slots, 1, row_dim), fill, dtype=store_dtype, device=device
).view(dtype)
for fill in (0x5A, 0xA5)
)
def assert_bytes_equal(actual, expected, label):
# Integer comparison preserves NaN encodings and the sign of zero.
torch.testing.assert_close(
actual.contiguous().view(torch.uint8),
expected.contiguous().view(torch.uint8),
rtol=0,
atol=0,
msg=lambda message: f"{label}: {message}",
)
@contextmanager
def assert_prefix_commit(
k, v, k_cache, v_cache, loc, lengths, k_scale, v_scale, *, inputs_mutated=False
):
"""Check the whole cache (including untouched slots) against eager division."""
rows = torch.arange(loc.shape[1], device=loc.device)
valid = (rows[None, :] < lengths[:, None]).reshape(-1)
src = torch.nonzero(valid, as_tuple=False).flatten()
dst = loc.reshape(-1)[src].long()
before = (k.clone(), v.clone())
scaled, expected = [], []
for source, cache, scale in zip(before, (k_cache, v_cache), (k_scale, v_scale)):
value = source.clone()
if scale is not None:
value.div_(scale)
scaled.append(value)
target = cache.contiguous().view(torch.uint8).clone()
target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src]
expected.append(target)
yield
for label, cache, reference in zip(
("K cache", "V cache"), (k_cache, v_cache), expected
):
assert_bytes_equal(cache, reference, label)
for label, source, reference in zip(
("K input", "V input"), (k, v), scaled if inputs_mutated else before
):
assert_bytes_equal(source, reference, label)