[qwen 3.8 next] Fuse QSA prefill KV gathering and dtype conversion (#41933)

This commit is contained in:
Qiaolin Yu
2026-10-10 22:40:23 -07:00
committed by GitHub
parent 5678b71b6c
commit 047405f534
3 changed files with 287 additions and 33 deletions
@@ -243,11 +243,8 @@ def _sparse_gqa_chunk_prefill(
mask=valid[:, None],
other=0.0,
)
# The chunk-prefill K/V tensors are gathered from the KV pool and can
# therefore carry the FP8 storage dtype, which Triton's dot rejects
# (`Unsupported rhs dtype fp8e4nv`). Convert to Q's dtype; the QSA
# backend writes the pool without per-tensor k/v scales, so this is a
# plain cast (no-op for BF16 pools).
# Direct callers can pass FP8 K/V; production packing uses Q's dtype.
# QSA stores unscaled K/V, so a plain cast supports both callers.
keys = keys.to(q_values.dtype)
values = values.to(q_values.dtype)
scores = tl.where(valid[None, :], tl.dot(q_values, keys), -float("inf"))
@@ -270,6 +267,105 @@ def _sparse_gqa_chunk_prefill(
)
@triton.jit(do_not_specialize=["TOTAL_K", "BS"])
def _pack_qsa_prefill_kv(
k,
v,
packed_k,
packed_v,
req_to_token,
req_indices,
cu_k,
SK0: tl.constexpr,
SK1: tl.constexpr,
SK2: tl.constexpr,
SV0: tl.constexpr,
SV1: tl.constexpr,
SV2: tl.constexpr,
SR0: tl.constexpr,
SR1: tl.constexpr,
HEADS: tl.constexpr,
DIM: tl.constexpr,
BLOCK: tl.constexpr,
TOTAL_K,
BS,
COMPACT: tl.constexpr,
):
offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
token = offsets // (HEADS * DIM)
if COMPACT:
valid = token < TOTAL_K
lo = tl.full((BLOCK,), 0, tl.int32)
hi = lo + BS
steps = BS
while steps > 0:
mid = (lo + hi + 1) // 2
start = tl.load(cu_k + mid).to(tl.int64)
after = token >= start
lo = tl.where(after, mid, lo)
hi = tl.where(after, hi, mid - 1)
steps = steps // 2
start = tl.load(cu_k + lo, valid, 0).to(tl.int64)
req = tl.load(req_indices + lo, valid, 0).to(tl.int64)
token = token - start
dst = offsets
else:
batch = tl.program_id(1)
start = tl.load(cu_k + batch).to(tl.int64)
end = tl.load(cu_k + batch + 1).to(tl.int64)
valid = token < end - start
req = tl.load(req_indices + batch).to(tl.int64)
dst = start * HEADS * DIM + offsets
head = offsets // DIM % HEADS
dim = offsets % DIM
slot = tl.load(req_to_token + req * SR0 + token * SR1, valid, 0).to(tl.int64)
keys = tl.load(k + slot * SK0 + head * SK1 + dim * SK2, valid, 0.0)
values = tl.load(v + slot * SV0 + head * SV1 + dim * SV2, valid, 0.0)
tl.store(packed_k + dst, keys, valid)
tl.store(packed_v + dst, values, valid)
def pack_qsa_prefill_kv(
*, k, v, req_to_token, req_indices, cu_k, total_k, max_k, output_dtype=None
):
heads, dim = k.shape[1:]
packed_k = torch.empty(
(total_k, heads, dim), dtype=output_dtype or k.dtype, device=k.device
)
packed_v = torch.empty(
(total_k, heads, dim), dtype=output_dtype or v.dtype, device=v.device
)
batch_size = req_indices.numel()
blocks_per_request = triton.cdiv(max_k * heads * dim, 1024)
compact = (
max_k * batch_size > 2 * total_k and blocks_per_request * batch_size >= 4096
)
grid = (
(triton.cdiv(total_k * heads * dim, 1024),)
if compact
else (blocks_per_request, batch_size)
)
_pack_qsa_prefill_kv[grid](
k,
v,
packed_k,
packed_v,
req_to_token,
req_indices,
cu_k,
*k.stride(),
*v.stride(),
*req_to_token.stride(),
HEADS=heads,
DIM=dim,
BLOCK=1024,
TOTAL_K=total_k,
BS=batch_size,
COMPACT=compact,
)
return packed_k, packed_v
def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
k, v = k.contiguous(), v.contiguous()
total_q, num_q_heads, head_dim = q.shape
@@ -565,6 +661,7 @@ def qwen_sparse_kv_extraction_compact_triton(
__all__ = [
"pack_qsa_prefill_kv",
"qwen_sparse_fa2_cu_seqlens_triton",
"qwen_sparse_valid_counts_triton",
"qwen_sparse_kv_extraction_compact_triton",
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.qsa.metadata import (
compressed_decode_view,
)
from sglang.srt.layers.attention.qsa.sparse_attn import (
pack_qsa_prefill_kv,
qwen_sparse_fa2_cu_seqlens_triton,
qwen_sparse_kv_extraction_compact_triton,
qwen_sparse_valid_counts_triton,
@@ -51,9 +52,6 @@ from sglang.srt.layers.cp.utils import (
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
)
from sglang.srt.runtime_context import get_spec
from sglang.srt.utils import is_hip
@@ -1508,38 +1506,26 @@ class QwenSparseAttnBackend(AttentionBackend):
)
return self._pad_extend_output(output, num_output_rows)
# The validated chunk-prefill kernel consumes tightly packed full-context
# K/V. Current-chunk K/V has already been committed to the cache above.
# Current-chunk K/V has already been committed to the cache above.
pool = self.token_to_kv_pool
k_buffer = pool.get_key_buffer(layer.layer_id)
v_buffer = pool.get_value_buffer(layer.layer_id)
req_to_token = self.req_to_token_pool.req_to_token
if is_in_breakable_cuda_graph():
# Reuse the prepared slot table to avoid a D2H sync per layer on replay.
req_to_token = self._resolve_metadata(forward_batch).token_slot_table
req_indices = range(len(sequence_lens))
else:
req_indices = forward_batch.req_pool_indices.tolist()
k_parts = [
k_buffer.index_select(
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
)
for i in range(len(sequence_lens))
]
v_parts = [
v_buffer.index_select(
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
)
for i in range(len(sequence_lens))
]
sequence_lens_tensor = torch.tensor(
sequence_lens, dtype=torch.int32, device=q.device
)
cu_seqlens_k = F.pad(sequence_lens_tensor.cumsum(0), (1, 0)).contiguous()
packed_k, packed_v = pack_qsa_prefill_kv(
k=pool.get_key_buffer(layer.layer_id),
v=pool.get_value_buffer(layer.layer_id),
req_to_token=self.req_to_token_pool.req_to_token,
req_indices=forward_batch.req_pool_indices,
cu_k=cu_seqlens_k,
total_k=sum(sequence_lens),
max_k=max(sequence_lens, default=1),
output_dtype=q.dtype,
)
output = sparse_gqa_fwd_interface_triton_ck(
q.contiguous(),
torch.cat(k_parts),
torch.cat(v_parts),
packed_k,
packed_v,
topk_indices,
cu_seqlens_q,
cu_seqlens_k,