[JIT] Fuse FP8 KV-cache quantization into the prefix-valid commit kernel (#40785)
Co-authored-by: Jayden Yu <jaydenyu@fb.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Jayden Yu
Xiaoyu Zhang
parent
f218798006
commit
cafb37a10d
@@ -57,6 +57,67 @@ def set_kv_buffer_prefix_valid_tiled(
|
||||
tl.store(dst_v_row_ptr, v_val, mask=mask_byte)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def set_kv_buffer_prefix_valid_tiled_fp8(
|
||||
src_k_ptr,
|
||||
src_v_ptr,
|
||||
dst_k_ptr,
|
||||
dst_v_ptr,
|
||||
loc_2d_ptr,
|
||||
commit_len_ptr,
|
||||
k_scale,
|
||||
v_scale,
|
||||
src_k_row_stride,
|
||||
src_v_row_stride,
|
||||
dst_k_row_stride,
|
||||
dst_v_row_stride,
|
||||
block_size,
|
||||
ROW_ELEMS: tl.constexpr,
|
||||
ELEMS_PER_TILE: tl.constexpr,
|
||||
K_SCALE_IS_TENSOR: tl.constexpr = False,
|
||||
V_SCALE_IS_TENSOR: tl.constexpr = False,
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
row = tl.program_id(1)
|
||||
tid = tl.program_id(2)
|
||||
|
||||
commit_len = tl.load(commit_len_ptr + bid)
|
||||
if row >= commit_len:
|
||||
return
|
||||
|
||||
elem_off = tid * ELEMS_PER_TILE + tl.arange(0, ELEMS_PER_TILE)
|
||||
mask_elem = elem_off < ROW_ELEMS
|
||||
|
||||
loc = tl.load(loc_2d_ptr + bid * block_size + row)
|
||||
src_row = bid * block_size + row
|
||||
|
||||
src_k_row_ptr = src_k_ptr + src_row * src_k_row_stride + elem_off
|
||||
src_v_row_ptr = src_v_ptr + src_row * src_v_row_stride + elem_off
|
||||
dst_k_row_ptr = dst_k_ptr + loc * dst_k_row_stride + elem_off
|
||||
dst_v_row_ptr = dst_v_ptr + loc * dst_v_row_stride + elem_off
|
||||
|
||||
# Tensor division rounds the divisor first and uses correctly rounded division.
|
||||
k_val = tl.load(src_k_row_ptr, mask=mask_elem, other=0).to(tl.float32)
|
||||
if K_SCALE_IS_TENSOR:
|
||||
k_scale = k_scale.to(src_k_ptr.dtype.element_ty).to(tl.float32)
|
||||
k_val = tl.div_rn(k_val, k_scale)
|
||||
else:
|
||||
# Eager host-scalar division multiplies by a rounded FP32 reciprocal.
|
||||
k_val = k_val * tl.div_rn(1.0, k_scale)
|
||||
k_val = k_val.to(src_k_ptr.dtype.element_ty).to(dst_k_ptr.dtype.element_ty)
|
||||
|
||||
v_val = tl.load(src_v_row_ptr, mask=mask_elem, other=0).to(tl.float32)
|
||||
if V_SCALE_IS_TENSOR:
|
||||
v_scale = v_scale.to(src_v_ptr.dtype.element_ty).to(tl.float32)
|
||||
v_val = tl.div_rn(v_val, v_scale)
|
||||
else:
|
||||
v_val = v_val * tl.div_rn(1.0, v_scale)
|
||||
v_val = v_val.to(src_v_ptr.dtype.element_ty).to(dst_v_ptr.dtype.element_ty)
|
||||
|
||||
tl.store(dst_k_row_ptr, k_val, mask=mask_elem)
|
||||
tl.store(dst_v_row_ptr, v_val, mask=mask_elem)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def copy_all_layer_kv_cache_tiled(
|
||||
data_ptrs,
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.kernels.ops.attention.dsa.quant_k_cache import (
|
||||
from sglang.kernels.ops.kvcache.cache_move import (
|
||||
copy_all_layer_kv_cache_func,
|
||||
set_kv_buffer_prefix_valid_tiled,
|
||||
set_kv_buffer_prefix_valid_tiled_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, is_fp8_fnuz
|
||||
@@ -271,6 +272,98 @@ def _set_kv_buffer_prefix_valid_impl(
|
||||
)
|
||||
|
||||
|
||||
def _set_kv_buffer_prefix_valid_impl_fp8(
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
k_scale: float,
|
||||
v_scale: float,
|
||||
loc_2d: torch.Tensor,
|
||||
commit_lens: torch.Tensor,
|
||||
row_dim: int,
|
||||
k_scale_is_tensor: bool = False,
|
||||
v_scale_is_tensor: bool = False,
|
||||
) -> None:
|
||||
if k.numel() == 0 or loc_2d.numel() == 0 or commit_lens.numel() == 0:
|
||||
return
|
||||
|
||||
if not k.is_contiguous():
|
||||
k = k.contiguous()
|
||||
if not v.is_contiguous():
|
||||
v = v.contiguous()
|
||||
if not loc_2d.is_contiguous():
|
||||
loc_2d = loc_2d.contiguous()
|
||||
if not commit_lens.is_contiguous():
|
||||
commit_lens = commit_lens.contiguous()
|
||||
|
||||
if row_dim <= 0:
|
||||
return
|
||||
|
||||
if row_dim >= 4096:
|
||||
elems_per_tile = 256
|
||||
num_warps = 8
|
||||
elif row_dim >= 2048:
|
||||
elems_per_tile = 128
|
||||
num_warps = 4
|
||||
else:
|
||||
elems_per_tile = 64
|
||||
num_warps = 4
|
||||
grid = (
|
||||
int(loc_2d.shape[0]),
|
||||
int(loc_2d.shape[1]),
|
||||
triton.cdiv(row_dim, elems_per_tile),
|
||||
)
|
||||
set_kv_buffer_prefix_valid_tiled_fp8[grid](
|
||||
k,
|
||||
v,
|
||||
k_cache,
|
||||
v_cache,
|
||||
loc_2d,
|
||||
commit_lens,
|
||||
k_scale,
|
||||
v_scale,
|
||||
int(k.stride(0)),
|
||||
int(v.stride(0)),
|
||||
int(k_cache.stride(0)),
|
||||
int(v_cache.stride(0)),
|
||||
int(loc_2d.shape[1]),
|
||||
ROW_ELEMS=row_dim,
|
||||
ELEMS_PER_TILE=elems_per_tile,
|
||||
K_SCALE_IS_TENSOR=k_scale_is_tensor,
|
||||
V_SCALE_IS_TENSOR=v_scale_is_tensor,
|
||||
num_warps=num_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fused_scale(
|
||||
scale,
|
||||
layer_scale,
|
||||
layer_scale_float,
|
||||
) -> Optional[float]:
|
||||
if isinstance(scale, (float, int)):
|
||||
return float(scale)
|
||||
|
||||
if (
|
||||
isinstance(scale, torch.Tensor)
|
||||
and scale.ndim == 0
|
||||
and scale.device.type == "cpu"
|
||||
):
|
||||
return float(scale.item())
|
||||
|
||||
if (
|
||||
isinstance(scale, torch.Tensor)
|
||||
and scale.ndim == 0
|
||||
and scale.is_cuda
|
||||
and scale is layer_scale
|
||||
and isinstance(layer_scale_float, (float, int))
|
||||
):
|
||||
return float(layer_scale_float)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class ReqToTokenPool:
|
||||
"""A memory pool that maps a request to its token locations."""
|
||||
|
||||
@@ -3104,21 +3197,6 @@ class MHATokenToKVPool(KVCache):
|
||||
f"{tuple(cache_k.shape)=} {tuple(cache_v.shape)=} {tuple(loc_2d.shape)=}."
|
||||
)
|
||||
|
||||
if cache_k.dtype != self.dtype:
|
||||
if k_scale is not None:
|
||||
cache_k.div_(k_scale)
|
||||
if v_scale is not None:
|
||||
cache_v.div_(v_scale)
|
||||
cache_k = cache_k.to(self.dtype)
|
||||
cache_v = cache_v.to(self.dtype)
|
||||
|
||||
if self.store_dtype != self.dtype:
|
||||
cache_k = cache_k.contiguous().view(self.store_dtype)
|
||||
cache_v = cache_v.contiguous().view(self.store_dtype)
|
||||
else:
|
||||
cache_k = cache_k.contiguous()
|
||||
cache_v = cache_v.contiguous()
|
||||
|
||||
if loc_2d.device != self.k_buffer[0].device:
|
||||
loc_2d = loc_2d.to(device=self.k_buffer[0].device, non_blocking=True)
|
||||
if commit_lens.device != self.k_buffer[0].device:
|
||||
@@ -3156,6 +3234,54 @@ class MHATokenToKVPool(KVCache):
|
||||
f"head_dim={self.head_dim} v_head_dim={self.v_head_dim}."
|
||||
)
|
||||
|
||||
fused_k_scale = _resolve_fused_scale(
|
||||
k_scale,
|
||||
getattr(layer, "k_scale", None),
|
||||
getattr(layer, "k_scale_float", None),
|
||||
)
|
||||
fused_v_scale = _resolve_fused_scale(
|
||||
v_scale,
|
||||
getattr(layer, "v_scale", None),
|
||||
getattr(layer, "v_scale_float", None),
|
||||
)
|
||||
# fuse quantization and writing into the same kernel
|
||||
if (
|
||||
cache_k.dtype != self.dtype
|
||||
and self.dtype == fp8_dtype
|
||||
and fused_k_scale is not None
|
||||
and fused_v_scale is not None
|
||||
):
|
||||
_set_kv_buffer_prefix_valid_impl_fp8(
|
||||
cache_k,
|
||||
cache_v,
|
||||
self.k_buffer[layer_id - self.start_layer].view(self.dtype),
|
||||
self.v_buffer[layer_id - self.start_layer].view(self.dtype),
|
||||
fused_k_scale,
|
||||
fused_v_scale,
|
||||
loc_2d,
|
||||
commit_lens,
|
||||
row_dim=self.row_dim,
|
||||
k_scale_is_tensor=isinstance(k_scale, torch.Tensor) and k_scale.is_cuda,
|
||||
v_scale_is_tensor=isinstance(v_scale, torch.Tensor) and v_scale.is_cuda,
|
||||
)
|
||||
return
|
||||
|
||||
# fallback: eager quantization
|
||||
if cache_k.dtype != self.dtype:
|
||||
if k_scale is not None:
|
||||
cache_k.div_(k_scale)
|
||||
if v_scale is not None:
|
||||
cache_v.div_(v_scale)
|
||||
cache_k = cache_k.to(self.dtype)
|
||||
cache_v = cache_v.to(self.dtype)
|
||||
|
||||
if self.store_dtype != self.dtype:
|
||||
cache_k = cache_k.contiguous().view(self.store_dtype)
|
||||
cache_v = cache_v.contiguous().view(self.store_dtype)
|
||||
else:
|
||||
cache_k = cache_k.contiguous()
|
||||
cache_v = cache_v.contiguous()
|
||||
|
||||
_set_kv_buffer_prefix_valid_impl(
|
||||
cache_k,
|
||||
cache_v,
|
||||
|
||||
@@ -316,6 +316,7 @@ class DFlashAttention(nn.Module):
|
||||
layer_id=layer_id,
|
||||
sliding_window_size=self.sliding_window_size,
|
||||
attn_type=self.attn_type,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
def forward_prepare_npu(self, positions, hidden_states):
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Shared eager oracle and byte assertions for prefix-valid KV-cache tests."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
|
||||
|
||||
|
||||
def make_kv_cache(total_slots, row_dim, dtype=fp8_dtype, device="cuda"):
|
||||
store_dtype = torch.uint8 if dtype == fp8_dtype else dtype
|
||||
return tuple(
|
||||
torch.full(
|
||||
(total_slots, 1, row_dim), fill, dtype=store_dtype, device=device
|
||||
).view(dtype)
|
||||
for fill in (0x5A, 0xA5)
|
||||
)
|
||||
|
||||
|
||||
def assert_bytes_equal(actual, expected, label):
|
||||
# Integer comparison preserves NaN encodings and the sign of zero.
|
||||
torch.testing.assert_close(
|
||||
actual.contiguous().view(torch.uint8),
|
||||
expected.contiguous().view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
msg=lambda message: f"{label}: {message}",
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def assert_prefix_commit(
|
||||
k, v, k_cache, v_cache, loc, lengths, k_scale, v_scale, *, inputs_mutated=False
|
||||
):
|
||||
"""Check the whole cache (including untouched slots) against eager division."""
|
||||
rows = torch.arange(loc.shape[1], device=loc.device)
|
||||
valid = (rows[None, :] < lengths[:, None]).reshape(-1)
|
||||
src = torch.nonzero(valid, as_tuple=False).flatten()
|
||||
dst = loc.reshape(-1)[src].long()
|
||||
before = (k.clone(), v.clone())
|
||||
scaled, expected = [], []
|
||||
for source, cache, scale in zip(before, (k_cache, v_cache), (k_scale, v_scale)):
|
||||
value = source.clone()
|
||||
if scale is not None:
|
||||
value.div_(scale)
|
||||
scaled.append(value)
|
||||
target = cache.contiguous().view(torch.uint8).clone()
|
||||
target[dst] = value.to(cache.dtype).contiguous().view(torch.uint8)[src]
|
||||
expected.append(target)
|
||||
|
||||
yield
|
||||
|
||||
for label, cache, reference in zip(
|
||||
("K cache", "V cache"), (k_cache, v_cache), expected
|
||||
):
|
||||
assert_bytes_equal(cache, reference, label)
|
||||
for label, source, reference in zip(
|
||||
("K input", "V input"), (k, v), scaled if inputs_mutated else before
|
||||
):
|
||||
assert_bytes_equal(source, reference, label)
|
||||
Reference in New Issue
Block a user