[DSA] Q8KV8 FP8 Sparse Prefill on GLM-5.3-Flash with BF16 KV Cache (#41790)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
29974d543e
commit
5100f8d11d
@@ -572,18 +572,22 @@ def _concat_cast_kv_fp8_pad_kernel(
|
||||
write zeros (the -1-sentinel landing pad the kernel's clamp maps to)."""
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs_n = tl.arange(0, NOPE)
|
||||
offs_r = tl.arange(0, ROPE)
|
||||
head = NOPE + ROPE
|
||||
if row < num_tokens:
|
||||
v_n = tl.load(k_ptr + row * k_stride + offs_n)
|
||||
tl.store(out_ptr + row * head + offs_n, v_n.to(tl.float8e4nv))
|
||||
v_r = tl.load(kr_ptr + row * kr_stride + offs_r)
|
||||
tl.store(out_ptr + row * head + NOPE + offs_r, v_r.to(tl.float8e4nv))
|
||||
# NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically.
|
||||
if ROPE > 0:
|
||||
offs_r = tl.arange(0, ROPE)
|
||||
v_r = tl.load(kr_ptr + row * kr_stride + offs_r)
|
||||
tl.store(out_ptr + row * head + NOPE + offs_r, v_r.to(tl.float8e4nv))
|
||||
else:
|
||||
zero_n = tl.zeros([NOPE], dtype=tl.float32).to(tl.float8e4nv)
|
||||
zero_r = tl.zeros([ROPE], dtype=tl.float32).to(tl.float8e4nv)
|
||||
tl.store(out_ptr + row * head + offs_n, zero_n)
|
||||
tl.store(out_ptr + row * head + NOPE + offs_r, zero_r)
|
||||
if ROPE > 0:
|
||||
offs_r = tl.arange(0, ROPE)
|
||||
zero_r = tl.zeros([ROPE], dtype=tl.float32).to(tl.float8e4nv)
|
||||
tl.store(out_ptr + row * head + NOPE + offs_r, zero_r)
|
||||
|
||||
|
||||
def concat_cast_kv_fp8_pad(
|
||||
@@ -598,7 +602,7 @@ def concat_cast_kv_fp8_pad(
|
||||
tail (3 kernels + one [tokens, 576] bf16 alloc). Same bf16->fp8
|
||||
store-cast the gather kernel uses (bit-identical bytes).
|
||||
|
||||
``out``: [total_rows, 576] fp8 slice (total_rows = num_tokens + pad band);
|
||||
``out``: [total_rows, NOPE+ROPE] fp8 slice (total_rows = num_tokens + pad band);
|
||||
``k``: [num_tokens, NOPE] bf16 view; ``k_rope``: [num_tokens, ROPE] bf16.
|
||||
"""
|
||||
total_rows, head = out.shape
|
||||
@@ -607,7 +611,7 @@ def concat_cast_kv_fp8_pad(
|
||||
assert head == nope + rope and out.dtype == torch.float8_e4m3fn
|
||||
k2 = k.view(num_tokens, nope)
|
||||
kr2 = k_rope.view(num_tokens, rope)
|
||||
assert k2.stride(-1) == 1 and kr2.stride(-1) == 1
|
||||
assert k2.stride(-1) == 1 and (rope == 0 or kr2.stride(-1) == 1)
|
||||
_concat_cast_kv_fp8_pad_kernel[(total_rows,)](
|
||||
out,
|
||||
k2,
|
||||
@@ -619,3 +623,95 @@ def concat_cast_kv_fp8_pad(
|
||||
ROPE=rope,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_cast_kv_bf16_paged_vec_kernel(
|
||||
kv_buffer_ptr,
|
||||
out_ptr,
|
||||
loc_ptr,
|
||||
num_tokens,
|
||||
total_rows,
|
||||
buffer_stride: tl.constexpr,
|
||||
out_stride: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
TOKENS_PER_PROG: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
rows = pid * TOKENS_PER_PROG + tl.arange(0, TOKENS_PER_PROG)
|
||||
gathered = rows < num_tokens
|
||||
loc = tl.load(loc_ptr + rows, mask=gathered, other=0).to(tl.int64)
|
||||
cols = tl.arange(0, HEAD_DIM)
|
||||
src = tl.load(
|
||||
kv_buffer_ptr + loc[:, None] * buffer_stride + cols[None, :],
|
||||
mask=gathered[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
# Rows past num_tokens load nothing and store that zero: they are the
|
||||
# -1-sentinel landing pad, so the trailing zero_ fuses into this kernel.
|
||||
tl.store(
|
||||
out_ptr + rows[:, None].to(tl.int64) * out_stride + cols[None, :],
|
||||
src,
|
||||
mask=(rows < total_rows)[:, None],
|
||||
)
|
||||
|
||||
|
||||
def gather_cast_kv_fp8_pad_paged(
|
||||
out: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
page_table_1_flattened: torch.Tensor,
|
||||
nope_dim: int,
|
||||
) -> torch.Tensor:
|
||||
"""Q8KV8 prefix KV prep for a bf16 MLA pool: gather rows [latent | rope] by
|
||||
token slot, store-cast them into the fp8 buffer, and zero the trailing
|
||||
-1-sentinel landing-pad rows.
|
||||
|
||||
``out``: [num_tokens + pad_rows, head_dim] fp8, contents arbitrary;
|
||||
``kv_cache``: [slots, 1, head_dim] bf16;
|
||||
``page_table_1_flattened``: [num_tokens] int32 slot ids.
|
||||
"""
|
||||
head_dim = out.shape[-1]
|
||||
num_tokens = page_table_1_flattened.shape[0]
|
||||
assert out.dtype == torch.float8_e4m3fn and out.is_contiguous()
|
||||
assert kv_cache.dtype == torch.bfloat16 and kv_cache.shape[-1] == head_dim
|
||||
# The gather addresses the source as loc * stride(0) + arange(0, nope_dim),
|
||||
# so only stride(0) is passed; a non-unit last-dim stride would silently read
|
||||
# the wrong elements. concat_cast_kv_fp8_pad asserts the same invariant.
|
||||
assert kv_cache.stride(-1) == 1
|
||||
if head_dim == nope_dim and head_dim & (head_dim - 1) == 0:
|
||||
# NoPE: source and destination rows are both one contiguous HEAD_DIM run,
|
||||
# so gather, cast and pad-zeroing fit in one kernel. The shared get
|
||||
# kernel below runs one program per token -- 131072 CTAs at a 128k
|
||||
# context -- and CTA count, not access width, is what binds there.
|
||||
total_rows = out.shape[0]
|
||||
_gather_cast_kv_bf16_paged_vec_kernel[
|
||||
(triton.cdiv(total_rows, _GATHER_TOKENS_PER_PROG),)
|
||||
](
|
||||
kv_cache,
|
||||
out,
|
||||
page_table_1_flattened,
|
||||
num_tokens,
|
||||
total_rows,
|
||||
kv_cache.stride(0),
|
||||
out.stride(0),
|
||||
HEAD_DIM=head_dim,
|
||||
TOKENS_PER_PROG=_GATHER_TOKENS_PER_PROG,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
# Local import: mla_buffer reaches into sglang.srt.runtime_context, and this
|
||||
# module lives under sglang.kernels; importing it at module scope would make
|
||||
# every kernels-side consumer pull in srt at import time.
|
||||
from sglang.kernels.ops.kvcache.mla_buffer import get_mla_kv_buffer_triton
|
||||
|
||||
if num_tokens > 0:
|
||||
rows = out[:num_tokens]
|
||||
get_mla_kv_buffer_triton(
|
||||
kv_cache,
|
||||
page_table_1_flattened,
|
||||
rows[:, :nope_dim],
|
||||
rows[:, nope_dim:] if head_dim > nope_dim else None,
|
||||
)
|
||||
out[num_tokens:].zero_()
|
||||
return out
|
||||
|
||||
@@ -390,9 +390,13 @@ def concat_and_cast_q_fp8_pad_kernel(
|
||||
no = tl.arange(0, NOPE)
|
||||
src_n = tl.load(q_nope_ptr + pid * nope_s0 + hr[:, None] * nope_s1 + no[None, :])
|
||||
tl.store(qpad_head + no[None, :], src_n)
|
||||
ro = tl.arange(0, ROPE)
|
||||
src_r = tl.load(q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :])
|
||||
tl.store(qpad_head + NOPE + ro[None, :], src_r)
|
||||
# NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically.
|
||||
if ROPE > 0:
|
||||
ro = tl.arange(0, ROPE)
|
||||
src_r = tl.load(
|
||||
q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :]
|
||||
)
|
||||
tl.store(qpad_head + NOPE + ro[None, :], src_r)
|
||||
|
||||
|
||||
def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads):
|
||||
@@ -400,7 +404,7 @@ def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads):
|
||||
[:, :num_heads, :] slice of the padded fp8 q buffer. Bit-exact replacement for the
|
||||
two strided converting copy_() in the Q8KV8 prefill q-prep, ~3.7x faster. Requires
|
||||
num_heads / nope_dim / rope_dim to be powers of two (always true for DeepSeek: 128
|
||||
heads / any TP, 512 nope, 64 rope)."""
|
||||
heads / any TP, 512 nope, 64 rope); rope_dim may also be 0 for NoPE models."""
|
||||
num_tokens = q_nope.shape[0]
|
||||
nope_dim = q_nope.shape[-1]
|
||||
rope_dim = q_rope.shape[-1]
|
||||
|
||||
@@ -39,6 +39,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
dsa_impl: _DSA_IMPL_T,
|
||||
phase: str,
|
||||
) -> None:
|
||||
# q8 prefill consumes the tail columns as ragged ids, like the history
|
||||
# picks. Only over a bfloat16 pool: gathering packed fp8 rows through
|
||||
# kpool tail columns is untested, so that combination keeps the guard.
|
||||
if (
|
||||
dsa_impl == "flashmla_sparse_q8"
|
||||
and phase == "prefill"
|
||||
and not self.dsa_kv_cache_store_fp8
|
||||
):
|
||||
return
|
||||
if (
|
||||
topk_indices is None
|
||||
or self.dsa_index_kpool <= 1
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
|
||||
@@ -28,6 +29,7 @@ logger = logging.getLogger(__name__)
|
||||
from sglang.kernels.ops.attention.dsa.dequant_k_cache import (
|
||||
concat_cast_kv_fp8_pad,
|
||||
dequantize_k_cache_paged,
|
||||
gather_cast_kv_fp8_pad_paged,
|
||||
gather_dequant_requant_fp8_paged,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsa.quant_k_cache import quantize_k_cache
|
||||
@@ -404,6 +406,77 @@ def _resolve_dsa_triton_union(*, union: int, deterministic: bool) -> int:
|
||||
return union
|
||||
|
||||
|
||||
def _validate_flashmla_sparse_q8_backend(
|
||||
*,
|
||||
prefill_impl: str,
|
||||
decode_impl: str,
|
||||
kv_cache_dtype: torch.dtype,
|
||||
kv_cache_store_fp8: bool,
|
||||
qk_rope_head_dim: int,
|
||||
device_sm_major: int,
|
||||
hisparse_enabled: bool = False,
|
||||
mixed_chunk_enabled: bool = False,
|
||||
) -> None:
|
||||
# flashmla_sparse_q8 casts q and KV to fp8 per call, so the KV cache may be
|
||||
# bfloat16 or fp8_e4m3.
|
||||
if prefill_impl == "flashmla_sparse_q8":
|
||||
if kv_cache_dtype not in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 computes attention in FP8 "
|
||||
"and reads a bfloat16 or fp8_e4m3 KV cache; "
|
||||
f"got kv_cache_dtype={kv_cache_dtype}."
|
||||
)
|
||||
# Not supported yet: an fp8_e4m3 pool on a NoPE model has not been validated
|
||||
# through the q8 prefix gather, and the KPool tail guard stays closed for
|
||||
# fp8 pools.
|
||||
if kv_cache_store_fp8 and qk_rope_head_dim == 0:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 does not support an "
|
||||
"fp8_e4m3 KV cache for NoPE MLA models (qk_rope_head_dim=0) yet; use "
|
||||
"--kv-cache-dtype bfloat16."
|
||||
)
|
||||
# An fp8 dtype that is NOT stored packed leaves the pool in the raw
|
||||
# 576-dim layout, which the bf16 prefix gather cannot read; without this
|
||||
# the failure is a message-less assert on the first prefix extend.
|
||||
if kv_cache_dtype == torch.float8_e4m3fn and not kv_cache_store_fp8:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 requires either a "
|
||||
"bfloat16 KV cache or the packed fp8_e4m3 pool; got an unpacked "
|
||||
"fp8_e4m3 pool (a trtllm backend keeps the raw layout). Use "
|
||||
"--kv-cache-dtype bfloat16."
|
||||
)
|
||||
if device_sm_major != 9:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 is SM90-only; got compute "
|
||||
f"capability sm_{device_sm_major}x."
|
||||
)
|
||||
# A mixed batch takes the PAGED top-k transform, which belongs to the
|
||||
# bf16 kernel. forward_extend refuses it, but the flag is readable here,
|
||||
# so the refusal belongs at launch rather than on the first mixed batch
|
||||
# formed under load.
|
||||
if mixed_chunk_enabled:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 does not support "
|
||||
"--enable-mixed-chunk: q8 prefill runs on RAGGED extend batches "
|
||||
"only."
|
||||
)
|
||||
# The q8 route hands top-k to the kernel as ragged ids and never binds the
|
||||
# page table HiSparse would translate, so the two cannot be combined.
|
||||
if hisparse_enabled:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 is not supported together "
|
||||
"with HiSparse: the q8 path consumes top-k as ragged ids and does "
|
||||
"not carry the HiSparse page translation."
|
||||
)
|
||||
if decode_impl == "flashmla_sparse_q8":
|
||||
raise ValueError(
|
||||
"--dsa-decode-backend flashmla_sparse_q8 is not supported: "
|
||||
"flashmla_sparse_q8 is a prefill-only backend. Pair it with a decode "
|
||||
"backend that reads the KV cache dtype (flashmla_kv for fp8_e4m3; "
|
||||
"tilelang or fa3 for bfloat16)."
|
||||
)
|
||||
|
||||
|
||||
class DeepseekSparseAttnBackend(
|
||||
DSAMetadataManagementMixin,
|
||||
DeepseekSparseAttnBackendKPoolMixin,
|
||||
@@ -576,23 +649,17 @@ class DeepseekSparseAttnBackend(
|
||||
self.device_sm_major = self.device_capability[0]
|
||||
self.kv_cache_dtype = model_runner.kv_cache_dtype
|
||||
|
||||
# `flashmla_sparse_q8` = the native FP8 SM90 sparse-prefill kernel. It always
|
||||
# runs FP8 (requires fp8_e4m3 KV) and is SM90-only, so validate both at
|
||||
# construction: an unsupported config must fail at launch rather than
|
||||
# mid-forward. `flashmla_sparse` remains the bf16 path with no such
|
||||
# requirement.
|
||||
if self.dsa_prefill_impl == "flashmla_sparse_q8":
|
||||
if self.kv_cache_dtype != torch.float8_e4m3fn:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 is native FP8 and requires "
|
||||
f"--kv-cache-dtype fp8_e4m3 (got kv_cache_dtype={self.kv_cache_dtype}); "
|
||||
"use --dsa-prefill-backend flashmla_sparse for the bf16 path."
|
||||
)
|
||||
if self.device_sm_major != 9:
|
||||
raise ValueError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 is SM90-only; got compute "
|
||||
f"capability sm_{self.device_sm_major}x."
|
||||
)
|
||||
# Unsupported flashmla_sparse_q8 configs must fail at launch, not mid-forward.
|
||||
_validate_flashmla_sparse_q8_backend(
|
||||
prefill_impl=self.dsa_prefill_impl,
|
||||
decode_impl=self.dsa_decode_impl,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_store_fp8=self.dsa_kv_cache_store_fp8,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
device_sm_major=self.device_sm_major,
|
||||
hisparse_enabled=self.hisparse_coordinator is not None,
|
||||
mixed_chunk_enabled=get_schedule().enable_mixed_chunk,
|
||||
)
|
||||
|
||||
if self.dsa_prefill_impl == "triton_sparse_mla":
|
||||
_validate_triton_sparse_mla_backend(
|
||||
@@ -602,21 +669,13 @@ class DeepseekSparseAttnBackend(
|
||||
index_kpool=self.dsa_index_kpool,
|
||||
)
|
||||
|
||||
# `flashmla_sparse_q8` is prefill-only (FP8 decode goes through
|
||||
# `flashmla_kv`); reject it as a decode backend, since argparse accepts it
|
||||
# via the shared CLI choices.
|
||||
if self.dsa_decode_impl == "flashmla_sparse_q8":
|
||||
raise ValueError(
|
||||
"--dsa-decode-backend flashmla_sparse_q8 is not supported: "
|
||||
"flashmla_sparse_q8 is a prefill-only backend. For FP8, use "
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 together with "
|
||||
"--dsa-decode-backend flashmla_kv."
|
||||
)
|
||||
|
||||
# Q8KV8 per-call device-tensor caches, populated lazily on the first
|
||||
# Q8KV8 dispatch (no-ops for other backends).
|
||||
self._q8kv8_identity_scale: Optional[torch.Tensor] = None
|
||||
self._q8kv8_qpad_buf: Optional[torch.Tensor] = None
|
||||
# Grow-only -1-filled top-k pad buffer; see _pad_topk_width_buffered.
|
||||
self._q8kv8_topk_pad_buf: Optional[torch.Tensor] = None
|
||||
self._q8kv8_topk_pad_width: Optional[int] = None
|
||||
# Persistent (grow-only) fp8 KV destination for the Q8KV8 prefill
|
||||
# gather: [capacity_rows, 576]. Avoids a fresh torch.zeros
|
||||
# (alloc + full-buffer FillFunctor) per layer per call; only the
|
||||
@@ -2141,6 +2200,18 @@ class DeepseekSparseAttnBackend(
|
||||
topk_transform_method = self.get_topk_transform_method(
|
||||
forward_batch.forward_mode
|
||||
)
|
||||
# q8 is only wired on the RAGGED route; PAGED would run the bf16 kernel.
|
||||
if (
|
||||
dsa_impl == "flashmla_sparse_q8"
|
||||
and topk_indices is not None
|
||||
and topk_transform_method != TopkTransformMethod.RAGGED
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"--dsa-prefill-backend flashmla_sparse_q8 supports EXTEND prefill "
|
||||
f"batches only; got forward_mode={forward_batch.forward_mode.name} "
|
||||
"(SPLIT_PREFILL under --enable-pdmux, or DLLM_EXTEND under "
|
||||
"--dllm-algorithm; --enable-mixed-chunk is rejected at launch)."
|
||||
)
|
||||
|
||||
if self.use_fused_topk:
|
||||
if topk_indices is not None:
|
||||
@@ -2226,11 +2297,9 @@ class DeepseekSparseAttnBackend(
|
||||
_has_prefix = any(forward_batch.extend_prefix_lens_cpu)
|
||||
page_table_1 = topk_indices
|
||||
|
||||
# `flashmla_sparse_q8` = native FP8 sparse prefill (constructor
|
||||
# guarantees fp8_e4m3 KV + SM90). The helper consumes q_nope/q_rope
|
||||
# directly (fusing the concat with the bf16->fp8 cast), so no bf16
|
||||
# q_all is materialized on this path. The prefix path hands over the
|
||||
# paged fp8 KV as-is; the non-prefix path passes the gathered bf16 KV.
|
||||
# flashmla_sparse_q8: FP8 compute on SM90 over a bf16 or fp8_e4m3 KV
|
||||
# cache. The helper fuses the q concat with the bf16->fp8 cast; prefix
|
||||
# batches hand over the paged cache as-is, others pass fresh bf16 KV.
|
||||
if dsa_impl == "flashmla_sparse_q8":
|
||||
if _has_prefix:
|
||||
page_table_1_flattened = (
|
||||
@@ -2809,6 +2878,45 @@ class DeepseekSparseAttnBackend(
|
||||
self._q8kv8_born_q_sentinel = buf
|
||||
return buf[:numel].view(num_tokens, num_heads, v_head_dim)
|
||||
|
||||
def _pad_topk_width_buffered(
|
||||
self, topk_indices: torch.Tensor, *, multiple: int
|
||||
) -> torch.Tensor:
|
||||
"""Widen top-k to a multiple of ``multiple``, padding with -1.
|
||||
|
||||
The destination is a persistent grow-only buffer rather than a fresh
|
||||
``torch.cat``: only the real columns change between calls, while a cat
|
||||
reallocates and rewrites the whole ``[num_tokens, padded_width]`` int32
|
||||
tensor on every DSA layer call (~71 MB at an 8192-token chunk). The pad
|
||||
columns are filled with -1 once at allocation and never written again,
|
||||
which is exactly what the SM90 kernel's -1 clamp reads. Eager-mode DSA
|
||||
runs layers sequentially on one stream, so single-buffer reuse is safe
|
||||
here for the same reason it is for _q8kv8_qpad_buf.
|
||||
"""
|
||||
assert topk_indices.dim() == 2
|
||||
width = topk_indices.shape[-1]
|
||||
pad = (-width) % multiple
|
||||
if pad == 0:
|
||||
return topk_indices
|
||||
rows = topk_indices.shape[0]
|
||||
buf = self._q8kv8_topk_pad_buf
|
||||
# A different REAL top-k width needs a fresh buffer even when it pads to
|
||||
# the same total: the tail is -1-filled only at allocation, so a buffer
|
||||
# built for a wider call still carries real ids in the columns this call
|
||||
# must read as padding. Comparing width + pad misses that, because 2051
|
||||
# and 2100 both pad to 2176.
|
||||
if (
|
||||
buf is None
|
||||
or buf.shape[0] < rows
|
||||
or buf.shape[-1] != width + pad
|
||||
or self._q8kv8_topk_pad_width != width
|
||||
):
|
||||
buf = topk_indices.new_full((rows, width + pad), -1)
|
||||
self._q8kv8_topk_pad_buf = buf
|
||||
self._q8kv8_topk_pad_width = width
|
||||
out = buf[:rows]
|
||||
out[:, :width].copy_(topk_indices)
|
||||
return out
|
||||
|
||||
def _forward_flashmla_sparse_q8kv8(
|
||||
self,
|
||||
q_nope: torch.Tensor,
|
||||
@@ -2827,21 +2935,26 @@ class DeepseekSparseAttnBackend(
|
||||
|
||||
Same contract as ``_forward_flashmla_sparse`` but executed through the
|
||||
FP8 ``sparse_mla_q8kv8_prefill_fwd`` kernel. Identity per-tensor
|
||||
scales (scalar 1.0) are used: a raw bf16->fp8 cast of q/kv is accurate
|
||||
on real DeepSeek-V3 magnitudes, so no dynamic rescaling is applied.
|
||||
scales (scalar 1.0) are used, matching the pre-existing fp8-pool path,
|
||||
which dequantizes by the group scale and re-casts at identity scale
|
||||
over the same range; no dynamic rescaling is applied. The Triton cast
|
||||
saturates at the fp8_e4m3 maximum (+-448) rather than producing NaN.
|
||||
The kernel runs via its fixed full-topk entry (``attn_sink`` /
|
||||
``topk_length`` left None), keeping control flow identical across DP
|
||||
ranks; -1 topk sentinels are clamped to distinct zero pad rows inside
|
||||
the kernel.
|
||||
|
||||
Two KV paths:
|
||||
KV paths:
|
||||
* non-prefix extend: ``kv_bf16`` (the gathered bf16 KV) is cast into
|
||||
a zero-padded fp8 buffer.
|
||||
* prefix extend: ``paged_kv_cache`` (fp8, 656 B/token: nope_fp8 +
|
||||
per-group scales + rope_bf16) is gathered, dequantized per group,
|
||||
and requantized to per-tensor fp8 in one fused Triton kernel
|
||||
(``gather_dequant_requant_fp8_paged``) — no intermediate bf16
|
||||
materialization.
|
||||
* prefix extend on an fp8 pool (656 B/token: nope_fp8 + per-group
|
||||
scales + rope_bf16): ``gather_dequant_requant_fp8_paged`` gathers,
|
||||
dequantizes per group and requantizes to per-tensor fp8 in one
|
||||
fused Triton kernel.
|
||||
* prefix extend on a bf16 pool: ``gather_cast_kv_fp8_pad_paged``
|
||||
gathers the rows and store-casts them to fp8.
|
||||
The top-k width is -1 padded to a multiple of 128 before the pad band
|
||||
is sized (kpool rows are index_topk + index_kpool - 1 wide).
|
||||
"""
|
||||
from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import (
|
||||
sparse_mla_q8kv8_prefill_fwd,
|
||||
@@ -2939,6 +3052,8 @@ class DeepseekSparseAttnBackend(
|
||||
# they need it EVERY call because a previous, larger call may have
|
||||
# left real KV data there. The gather kernel fuses the pad-row
|
||||
# zeroing; the bf16 path zeroes the tail explicitly.
|
||||
# The kernel walks top-k in pairs of 64-wide blocks (topk % 128 == 0).
|
||||
page_table_1 = self._pad_topk_width_buffered(page_table_1, multiple=128)
|
||||
topk = page_table_1.shape[-1]
|
||||
if paged_kv_cache is not None:
|
||||
num_kv_tokens = page_table_1_flattened.shape[0]
|
||||
@@ -2955,13 +3070,20 @@ class DeepseekSparseAttnBackend(
|
||||
device=dev,
|
||||
)
|
||||
self._q8kv8_kv_buf = kv_buf
|
||||
if paged_kv_cache is not None:
|
||||
if paged_kv_cache is not None and self.dsa_kv_cache_store_fp8:
|
||||
kv_padded = gather_dequant_requant_fp8_paged(
|
||||
paged_kv_cache,
|
||||
page_table_1_flattened,
|
||||
extra_rows=topk,
|
||||
out=kv_buf[:total_kv_rows],
|
||||
).view(-1, 1, head_dim)
|
||||
elif paged_kv_cache is not None:
|
||||
kv_padded = gather_cast_kv_fp8_pad_paged(
|
||||
out=kv_buf[:total_kv_rows],
|
||||
kv_cache=paged_kv_cache,
|
||||
page_table_1_flattened=page_table_1_flattened,
|
||||
nope_dim=self.kv_lora_rank,
|
||||
).view(-1, 1, head_dim)
|
||||
elif kv_k is not None:
|
||||
# Fused non-prefix KV prep (SGLANG_ENABLE_DSA_Q8KV8_KV_CAT_FUSION):
|
||||
# cast-concat k/k_rope straight into the fp8 buffer + zero the pad
|
||||
@@ -3831,13 +3953,14 @@ class DeepseekSparseAttnBackend(
|
||||
"""
|
||||
if (
|
||||
# disable for MTP
|
||||
self.dsa_kv_cache_store_fp8
|
||||
# flashmla_sparse_q8 shares flashmla_sparse's RAGGED prefill routing — the q8
|
||||
# dispatch lives inside the RAGGED branch of forward_extend; without this the
|
||||
# transform is PAGED, the q8 path is skipped, and the bf16 kernel crashes on
|
||||
# fp8 KV ("kv must have dtype kBFloat16"). triton_sparse_mla is listed for
|
||||
# the same reason: its dispatch also dequantizes inside the RAGGED branch
|
||||
# and its kernel is bf16-only.
|
||||
(
|
||||
self.dsa_kv_cache_store_fp8
|
||||
or self.dsa_prefill_impl == "flashmla_sparse_q8"
|
||||
)
|
||||
# flashmla_sparse_q8 always gathers KV into a contiguous fp8 buffer;
|
||||
# it is RAGGED whatever the pool dtype, since its dispatch lives there.
|
||||
# triton_sparse_mla is listed for an fp8 pool: its dispatch also
|
||||
# dequantizes inside the RAGGED branch and its kernel is bf16-only.
|
||||
and self.dsa_prefill_impl
|
||||
in ("flashmla_sparse", "flashmla_sparse_q8", "triton_sparse_mla")
|
||||
and forward_mode == ForwardMode.EXTEND
|
||||
|
||||
Reference in New Issue
Block a user