diff --git a/python/sglang/srt/layers/attention/qsa/sparse_attn.py b/python/sglang/srt/layers/attention/qsa/sparse_attn.py index a52c1e510a..2aca96d711 100644 --- a/python/sglang/srt/layers/attention/qsa/sparse_attn.py +++ b/python/sglang/srt/layers/attention/qsa/sparse_attn.py @@ -243,11 +243,8 @@ def _sparse_gqa_chunk_prefill( mask=valid[:, None], other=0.0, ) - # The chunk-prefill K/V tensors are gathered from the KV pool and can - # therefore carry the FP8 storage dtype, which Triton's dot rejects - # (`Unsupported rhs dtype fp8e4nv`). Convert to Q's dtype; the QSA - # backend writes the pool without per-tensor k/v scales, so this is a - # plain cast (no-op for BF16 pools). + # Direct callers can pass FP8 K/V; production packing uses Q's dtype. + # QSA stores unscaled K/V, so a plain cast supports both callers. keys = keys.to(q_values.dtype) values = values.to(q_values.dtype) scores = tl.where(valid[None, :], tl.dot(q_values, keys), -float("inf")) @@ -270,6 +267,105 @@ def _sparse_gqa_chunk_prefill( ) +@triton.jit(do_not_specialize=["TOTAL_K", "BS"]) +def _pack_qsa_prefill_kv( + k, + v, + packed_k, + packed_v, + req_to_token, + req_indices, + cu_k, + SK0: tl.constexpr, + SK1: tl.constexpr, + SK2: tl.constexpr, + SV0: tl.constexpr, + SV1: tl.constexpr, + SV2: tl.constexpr, + SR0: tl.constexpr, + SR1: tl.constexpr, + HEADS: tl.constexpr, + DIM: tl.constexpr, + BLOCK: tl.constexpr, + TOTAL_K, + BS, + COMPACT: tl.constexpr, +): + offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + token = offsets // (HEADS * DIM) + if COMPACT: + valid = token < TOTAL_K + lo = tl.full((BLOCK,), 0, tl.int32) + hi = lo + BS + steps = BS + while steps > 0: + mid = (lo + hi + 1) // 2 + start = tl.load(cu_k + mid).to(tl.int64) + after = token >= start + lo = tl.where(after, mid, lo) + hi = tl.where(after, hi, mid - 1) + steps = steps // 2 + start = tl.load(cu_k + lo, valid, 0).to(tl.int64) + req = tl.load(req_indices + lo, valid, 0).to(tl.int64) + token = token - start + dst = offsets + else: + batch = tl.program_id(1) + start = tl.load(cu_k + batch).to(tl.int64) + end = tl.load(cu_k + batch + 1).to(tl.int64) + valid = token < end - start + req = tl.load(req_indices + batch).to(tl.int64) + dst = start * HEADS * DIM + offsets + head = offsets // DIM % HEADS + dim = offsets % DIM + slot = tl.load(req_to_token + req * SR0 + token * SR1, valid, 0).to(tl.int64) + keys = tl.load(k + slot * SK0 + head * SK1 + dim * SK2, valid, 0.0) + values = tl.load(v + slot * SV0 + head * SV1 + dim * SV2, valid, 0.0) + tl.store(packed_k + dst, keys, valid) + tl.store(packed_v + dst, values, valid) + + +def pack_qsa_prefill_kv( + *, k, v, req_to_token, req_indices, cu_k, total_k, max_k, output_dtype=None +): + heads, dim = k.shape[1:] + packed_k = torch.empty( + (total_k, heads, dim), dtype=output_dtype or k.dtype, device=k.device + ) + packed_v = torch.empty( + (total_k, heads, dim), dtype=output_dtype or v.dtype, device=v.device + ) + batch_size = req_indices.numel() + blocks_per_request = triton.cdiv(max_k * heads * dim, 1024) + compact = ( + max_k * batch_size > 2 * total_k and blocks_per_request * batch_size >= 4096 + ) + grid = ( + (triton.cdiv(total_k * heads * dim, 1024),) + if compact + else (blocks_per_request, batch_size) + ) + _pack_qsa_prefill_kv[grid]( + k, + v, + packed_k, + packed_v, + req_to_token, + req_indices, + cu_k, + *k.stride(), + *v.stride(), + *req_to_token.stride(), + HEADS=heads, + DIM=dim, + BLOCK=1024, + TOTAL_K=total_k, + BS=batch_size, + COMPACT=compact, + ) + return packed_k, packed_v + + def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale): k, v = k.contiguous(), v.contiguous() total_q, num_q_heads, head_dim = q.shape @@ -565,6 +661,7 @@ def qwen_sparse_kv_extraction_compact_triton( __all__ = [ + "pack_qsa_prefill_kv", "qwen_sparse_fa2_cu_seqlens_triton", "qwen_sparse_valid_counts_triton", "qwen_sparse_kv_extraction_compact_triton", diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 7acae5284d..6eabce2797 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -34,6 +34,7 @@ from sglang.srt.layers.attention.qsa.metadata import ( compressed_decode_view, ) from sglang.srt.layers.attention.qsa.sparse_attn import ( + pack_qsa_prefill_kv, qwen_sparse_fa2_cu_seqlens_triton, qwen_sparse_kv_extraction_compact_triton, qwen_sparse_valid_counts_triton, @@ -51,9 +52,6 @@ from sglang.srt.layers.cp.utils import ( from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( - is_in_breakable_cuda_graph, -) from sglang.srt.runtime_context import get_spec from sglang.srt.utils import is_hip @@ -1508,38 +1506,26 @@ class QwenSparseAttnBackend(AttentionBackend): ) return self._pad_extend_output(output, num_output_rows) - # The validated chunk-prefill kernel consumes tightly packed full-context - # K/V. Current-chunk K/V has already been committed to the cache above. + # Current-chunk K/V has already been committed to the cache above. pool = self.token_to_kv_pool - k_buffer = pool.get_key_buffer(layer.layer_id) - v_buffer = pool.get_value_buffer(layer.layer_id) - req_to_token = self.req_to_token_pool.req_to_token - if is_in_breakable_cuda_graph(): - # Reuse the prepared slot table to avoid a D2H sync per layer on replay. - req_to_token = self._resolve_metadata(forward_batch).token_slot_table - req_indices = range(len(sequence_lens)) - else: - req_indices = forward_batch.req_pool_indices.tolist() - k_parts = [ - k_buffer.index_select( - 0, req_to_token[req_indices[i], : sequence_lens[i]].long() - ) - for i in range(len(sequence_lens)) - ] - v_parts = [ - v_buffer.index_select( - 0, req_to_token[req_indices[i], : sequence_lens[i]].long() - ) - for i in range(len(sequence_lens)) - ] sequence_lens_tensor = torch.tensor( sequence_lens, dtype=torch.int32, device=q.device ) cu_seqlens_k = F.pad(sequence_lens_tensor.cumsum(0), (1, 0)).contiguous() + packed_k, packed_v = pack_qsa_prefill_kv( + k=pool.get_key_buffer(layer.layer_id), + v=pool.get_value_buffer(layer.layer_id), + req_to_token=self.req_to_token_pool.req_to_token, + req_indices=forward_batch.req_pool_indices, + cu_k=cu_seqlens_k, + total_k=sum(sequence_lens), + max_k=max(sequence_lens, default=1), + output_dtype=q.dtype, + ) output = sparse_gqa_fwd_interface_triton_ck( q.contiguous(), - torch.cat(k_parts), - torch.cat(v_parts), + packed_k, + packed_v, topk_indices, cu_seqlens_q, cu_seqlens_k, diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa.py b/test/registered/kernels/ops/attention/qsa/test_qsa.py index 60858ae05c..e7b6a39024 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa.py @@ -1,4 +1,5 @@ import sys +from itertools import accumulate from types import MethodType, ModuleType, SimpleNamespace import pytest @@ -25,6 +26,7 @@ from sglang.srt.layers.attention.qsa.mqa import ( ) from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer from sglang.srt.layers.attention.qsa.sparse_attn import ( + pack_qsa_prefill_kv, qwen_sparse_fa2_cu_seqlens_triton, qwen_sparse_kv_extraction_compact_triton, sparse_gqa_fwd_interface_triton_ck, @@ -1640,5 +1642,174 @@ def test_qsa_draft_metadata_multi_step_graph(bs, padding): ) +@pytest.mark.parametrize("strided", [False, True]) +@pytest.mark.parametrize( + "dtype,output_dtype", + [ + (torch.bfloat16, None), + (torch.float8_e4m3fn, None), + (torch.float8_e4m3fn, torch.bfloat16), + ], +) +@pytest.mark.parametrize("head_dim,num_kv_heads,group_size", [(128, 2, 4), (256, 1, 6)]) +@pytest.mark.parametrize( + "query_lens,prefix_lens", + [ + ([1, 7], [31, 19]), + ([17, 33], [23, 0]), + ([2049, 3], [7, 40]), + ([3, 1, 2], [1018, 128, 128]), + ([3, 1, 2], [253, 63, 62]), + ([3, 1, 2], [254, 63, 62]), + ([3, 1, 2], [8189, 2047, 2046]), + ([3, 1, 2], [8190, 2047, 2046]), + ], +) +def test_qsa_prefill_kv_packing( + dtype, + output_dtype, + head_dim, + num_kv_heads, + group_size, + query_lens, + prefix_lens, + strided, +): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9): + pytest.skip("FP8-capable CUDA GPU required") + torch.manual_seed(42) + device = "cuda" + lens = [q + p for q, p in zip(query_lens, prefix_lens)] + request_ids = [3, 1, 0][: len(lens)] + replay_ids = [2, 0, 3][: len(lens)] + table_width = max(lens) + 11 + pool_size = 16 * table_width + table = torch.randperm(pool_size, device=device).reshape(8, table_width * 2)[ + ::2, ::2 + ] + if strided: + table = table.to(torch.int32) + req_indices = torch.tensor(request_ids, device=device, dtype=torch.int64) + k = torch.randn( + pool_size, num_kv_heads, head_dim, device=device, dtype=torch.bfloat16 + ).to(dtype) + v = torch.randn_like(k.to(torch.bfloat16)).to(dtype) + if strided: + k_storage = torch.empty( + pool_size, num_kv_heads, head_dim * 2, device=device, dtype=dtype + ) + v_storage = torch.empty_like(k_storage) + k = k_storage[..., ::2].copy_(k) + v = v_storage[..., ::2].copy_(v) + q = torch.randn( + sum(query_lens), + num_kv_heads * group_size, + head_dim, + device=device, + dtype=torch.bfloat16, + ) + cu_q = torch.tensor([0, *accumulate(query_lens)], device=device, dtype=torch.int32) + cu_k = torch.tensor([0, *accumulate(lens)], device=device, dtype=torch.int32) + kv_lens = torch.tensor(lens, device=device, dtype=torch.int32) + indices = torch.full((sum(query_lens), 32), -1, device=device, dtype=torch.int32) + row = 0 + for count, prefix in zip(query_lens, prefix_lens): + for offset in range(count): + visible = prefix + offset + 1 + selected = torch.randperm(visible, device=device)[:32] + indices[row, : selected.numel()] = selected + row += 1 + + def reference(request_ids): + slots = torch.cat([table[request_ids[i], :n] for i, n in enumerate(lens)]) + return sparse_gqa_fwd_interface_triton_ck( + q, + k.index_select(0, slots).to(output_dtype or k.dtype), + v.index_select(0, slots).to(output_dtype or v.dtype), + indices, + cu_q, + cu_k, + kv_lens, + head_dim**-0.5, + ) + + def packed(): + packed_k, packed_v = pack_qsa_prefill_kv( + k=k, + v=v, + req_to_token=table, + req_indices=req_indices, + cu_k=cu_k, + total_k=sum(lens), + max_k=max(lens), + output_dtype=output_dtype, + ) + if not torch.cuda.is_current_stream_capturing(): + slots = torch.cat([table[req_indices[i], :n] for i, n in enumerate(lens)]) + torch.testing.assert_close( + packed_k.float(), k.index_select(0, slots).float(), rtol=0, atol=0 + ) + torch.testing.assert_close( + packed_v.float(), v.index_select(0, slots).float(), rtol=0, atol=0 + ) + return packed_k, packed_v + + def attend(packed_k, packed_v): + return sparse_gqa_fwd_interface_triton_ck( + q, + packed_k, + packed_v, + indices, + cu_q, + cu_k, + kv_lens, + head_dim**-0.5, + ) + + expected = reference(request_ids) + actual = attend(*packed()) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = packed() + req_indices.copy_(torch.tensor(replay_ids, device=device)) + expected = reference(replay_ids) + graph.replay() + torch.testing.assert_close(attend(*captured), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("head_dim", [128, 256]) +@pytest.mark.parametrize("heads", [1, 3, 6, 12]) +def test_qsa_large_fp8_prefill_float_reference(head_dim, heads): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9): + pytest.skip("FP8-capable CUDA GPU required") + torch.manual_seed(19) + rows, prefix, topk = 2053, 64, 64 + q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16) + k = torch.randn(rows + prefix, 1, head_dim, device="cuda", dtype=torch.bfloat16).to( + torch.float8_e4m3fn + ) + v = torch.randn_like(k.to(torch.bfloat16)).to(k.dtype) + visible = torch.arange(rows, device="cuda") + prefix + 1 + indices = ( + torch.arange(topk, device="cuda")[None, :] + + torch.arange(rows, device="cuda")[:, None] * 7 + ) % visible[:, None] + indices = indices.to(torch.int32) + cu_q = torch.tensor([0, rows], device="cuda", dtype=torch.int32) + cu_k = torch.tensor([0, rows + prefix], device="cuda", dtype=torch.int32) + lengths = cu_k[1:] + actual = sparse_gqa_fwd_interface_triton_ck( + q, k, v, indices, cu_q, cu_k, lengths, head_dim**-0.5 + ) + for row in [0, 17, rows // 2, rows - 1]: + keys = k.index_select(0, indices[row].long()).squeeze(1).double() + values = v.index_select(0, indices[row].long()).squeeze(1).double() + expected = ( + torch.softmax(q[row].double() @ keys.T * head_dim**-0.5, dim=-1) @ values + ) + torch.testing.assert_close(actual[row].double(), expected, rtol=2e-2, atol=2e-2) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))