[qwen 3.8 next] Fuse QSA prefill KV gathering and dtype conversion (#41933)
This commit is contained in:
@@ -243,11 +243,8 @@ def _sparse_gqa_chunk_prefill(
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
# The chunk-prefill K/V tensors are gathered from the KV pool and can
|
||||
# therefore carry the FP8 storage dtype, which Triton's dot rejects
|
||||
# (`Unsupported rhs dtype fp8e4nv`). Convert to Q's dtype; the QSA
|
||||
# backend writes the pool without per-tensor k/v scales, so this is a
|
||||
# plain cast (no-op for BF16 pools).
|
||||
# Direct callers can pass FP8 K/V; production packing uses Q's dtype.
|
||||
# QSA stores unscaled K/V, so a plain cast supports both callers.
|
||||
keys = keys.to(q_values.dtype)
|
||||
values = values.to(q_values.dtype)
|
||||
scores = tl.where(valid[None, :], tl.dot(q_values, keys), -float("inf"))
|
||||
@@ -270,6 +267,105 @@ def _sparse_gqa_chunk_prefill(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["TOTAL_K", "BS"])
|
||||
def _pack_qsa_prefill_kv(
|
||||
k,
|
||||
v,
|
||||
packed_k,
|
||||
packed_v,
|
||||
req_to_token,
|
||||
req_indices,
|
||||
cu_k,
|
||||
SK0: tl.constexpr,
|
||||
SK1: tl.constexpr,
|
||||
SK2: tl.constexpr,
|
||||
SV0: tl.constexpr,
|
||||
SV1: tl.constexpr,
|
||||
SV2: tl.constexpr,
|
||||
SR0: tl.constexpr,
|
||||
SR1: tl.constexpr,
|
||||
HEADS: tl.constexpr,
|
||||
DIM: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
TOTAL_K,
|
||||
BS,
|
||||
COMPACT: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
|
||||
token = offsets // (HEADS * DIM)
|
||||
if COMPACT:
|
||||
valid = token < TOTAL_K
|
||||
lo = tl.full((BLOCK,), 0, tl.int32)
|
||||
hi = lo + BS
|
||||
steps = BS
|
||||
while steps > 0:
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(cu_k + mid).to(tl.int64)
|
||||
after = token >= start
|
||||
lo = tl.where(after, mid, lo)
|
||||
hi = tl.where(after, hi, mid - 1)
|
||||
steps = steps // 2
|
||||
start = tl.load(cu_k + lo, valid, 0).to(tl.int64)
|
||||
req = tl.load(req_indices + lo, valid, 0).to(tl.int64)
|
||||
token = token - start
|
||||
dst = offsets
|
||||
else:
|
||||
batch = tl.program_id(1)
|
||||
start = tl.load(cu_k + batch).to(tl.int64)
|
||||
end = tl.load(cu_k + batch + 1).to(tl.int64)
|
||||
valid = token < end - start
|
||||
req = tl.load(req_indices + batch).to(tl.int64)
|
||||
dst = start * HEADS * DIM + offsets
|
||||
head = offsets // DIM % HEADS
|
||||
dim = offsets % DIM
|
||||
slot = tl.load(req_to_token + req * SR0 + token * SR1, valid, 0).to(tl.int64)
|
||||
keys = tl.load(k + slot * SK0 + head * SK1 + dim * SK2, valid, 0.0)
|
||||
values = tl.load(v + slot * SV0 + head * SV1 + dim * SV2, valid, 0.0)
|
||||
tl.store(packed_k + dst, keys, valid)
|
||||
tl.store(packed_v + dst, values, valid)
|
||||
|
||||
|
||||
def pack_qsa_prefill_kv(
|
||||
*, k, v, req_to_token, req_indices, cu_k, total_k, max_k, output_dtype=None
|
||||
):
|
||||
heads, dim = k.shape[1:]
|
||||
packed_k = torch.empty(
|
||||
(total_k, heads, dim), dtype=output_dtype or k.dtype, device=k.device
|
||||
)
|
||||
packed_v = torch.empty(
|
||||
(total_k, heads, dim), dtype=output_dtype or v.dtype, device=v.device
|
||||
)
|
||||
batch_size = req_indices.numel()
|
||||
blocks_per_request = triton.cdiv(max_k * heads * dim, 1024)
|
||||
compact = (
|
||||
max_k * batch_size > 2 * total_k and blocks_per_request * batch_size >= 4096
|
||||
)
|
||||
grid = (
|
||||
(triton.cdiv(total_k * heads * dim, 1024),)
|
||||
if compact
|
||||
else (blocks_per_request, batch_size)
|
||||
)
|
||||
_pack_qsa_prefill_kv[grid](
|
||||
k,
|
||||
v,
|
||||
packed_k,
|
||||
packed_v,
|
||||
req_to_token,
|
||||
req_indices,
|
||||
cu_k,
|
||||
*k.stride(),
|
||||
*v.stride(),
|
||||
*req_to_token.stride(),
|
||||
HEADS=heads,
|
||||
DIM=dim,
|
||||
BLOCK=1024,
|
||||
TOTAL_K=total_k,
|
||||
BS=batch_size,
|
||||
COMPACT=compact,
|
||||
)
|
||||
return packed_k, packed_v
|
||||
|
||||
|
||||
def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
|
||||
k, v = k.contiguous(), v.contiguous()
|
||||
total_q, num_q_heads, head_dim = q.shape
|
||||
@@ -565,6 +661,7 @@ def qwen_sparse_kv_extraction_compact_triton(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"pack_qsa_prefill_kv",
|
||||
"qwen_sparse_fa2_cu_seqlens_triton",
|
||||
"qwen_sparse_valid_counts_triton",
|
||||
"qwen_sparse_kv_extraction_compact_triton",
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.qsa.metadata import (
|
||||
compressed_decode_view,
|
||||
)
|
||||
from sglang.srt.layers.attention.qsa.sparse_attn import (
|
||||
pack_qsa_prefill_kv,
|
||||
qwen_sparse_fa2_cu_seqlens_triton,
|
||||
qwen_sparse_kv_extraction_compact_triton,
|
||||
qwen_sparse_valid_counts_triton,
|
||||
@@ -51,9 +52,6 @@ from sglang.srt.layers.cp.utils import (
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
@@ -1508,38 +1506,26 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
)
|
||||
return self._pad_extend_output(output, num_output_rows)
|
||||
|
||||
# The validated chunk-prefill kernel consumes tightly packed full-context
|
||||
# K/V. Current-chunk K/V has already been committed to the cache above.
|
||||
# Current-chunk K/V has already been committed to the cache above.
|
||||
pool = self.token_to_kv_pool
|
||||
k_buffer = pool.get_key_buffer(layer.layer_id)
|
||||
v_buffer = pool.get_value_buffer(layer.layer_id)
|
||||
req_to_token = self.req_to_token_pool.req_to_token
|
||||
if is_in_breakable_cuda_graph():
|
||||
# Reuse the prepared slot table to avoid a D2H sync per layer on replay.
|
||||
req_to_token = self._resolve_metadata(forward_batch).token_slot_table
|
||||
req_indices = range(len(sequence_lens))
|
||||
else:
|
||||
req_indices = forward_batch.req_pool_indices.tolist()
|
||||
k_parts = [
|
||||
k_buffer.index_select(
|
||||
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
|
||||
)
|
||||
for i in range(len(sequence_lens))
|
||||
]
|
||||
v_parts = [
|
||||
v_buffer.index_select(
|
||||
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
|
||||
)
|
||||
for i in range(len(sequence_lens))
|
||||
]
|
||||
sequence_lens_tensor = torch.tensor(
|
||||
sequence_lens, dtype=torch.int32, device=q.device
|
||||
)
|
||||
cu_seqlens_k = F.pad(sequence_lens_tensor.cumsum(0), (1, 0)).contiguous()
|
||||
packed_k, packed_v = pack_qsa_prefill_kv(
|
||||
k=pool.get_key_buffer(layer.layer_id),
|
||||
v=pool.get_value_buffer(layer.layer_id),
|
||||
req_to_token=self.req_to_token_pool.req_to_token,
|
||||
req_indices=forward_batch.req_pool_indices,
|
||||
cu_k=cu_seqlens_k,
|
||||
total_k=sum(sequence_lens),
|
||||
max_k=max(sequence_lens, default=1),
|
||||
output_dtype=q.dtype,
|
||||
)
|
||||
output = sparse_gqa_fwd_interface_triton_ck(
|
||||
q.contiguous(),
|
||||
torch.cat(k_parts),
|
||||
torch.cat(v_parts),
|
||||
packed_k,
|
||||
packed_v,
|
||||
topk_indices,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_k,
|
||||
|
||||
Reference in New Issue
Block a user