From f70e8c68fc763a263244d1285cd09cbf3312ad06 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Sun, 4 Oct 2026 17:19:14 -0700 Subject: [PATCH] [Dsv4.1] Bounded replay with sparse mla path (#42273) --- .../attention/dsv4/sparse_prefill_kernels.py | 23 ++- .../layers/attention/deepseek_v4_backend.py | 31 +++- .../attention/dsv4/sparse_prefill_utils.py | 54 +++++-- .../ops/attention/test_flashmla_sched_meta.py | 22 +++ .../test_q8kv8_sparse_prefill_backend.py | 138 +++++++++++++++++- 5 files changed, 247 insertions(+), 21 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py index 558dc9c901..89e064de85 100644 --- a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py @@ -49,9 +49,13 @@ def _combine_topk_swa_indices_kernel( gather_lens_ptr, compressed_base_ptr, swa_base_ptr, + swa_indices_ptr, + swa_indices_stride, + swa_lengths_ptr, num_tokens, num_reqs, top_k, + EXPLICIT_SWA: tl.constexpr, COMPRESS_RATIO: tl.constexpr, WINDOW_SIZE: tl.constexpr, WIDTH: tl.constexpr, @@ -92,7 +96,10 @@ def _combine_topk_swa_indices_kernel( # -1 entries inside the top-k span stay -1 (attention skips them). # top_k=0 disables the compressed portion for SWA-only layers. topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0) - swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0) + if EXPLICIT_SWA: + swa_len = tl.load(swa_lengths_ptr + token, mask=owned, other=0) + else: + swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0) combined_row = token.to(tl.int64)[:, None] * combined_indices_stride topk_row = token.to(tl.int64)[:, None] * topk_indices_stride @@ -107,9 +114,19 @@ def _combine_topk_swa_indices_kernel( # Workspace SWA index: swa_base[r] + (gather_offset_in_buffer). # For positions [pos - swa_len + 1, pos], the buffer offsets are # [pos - swa_len + 1 - gather_start, pos - gather_start]. - swa_start = swa_base + pos - swa_len + 1 - gather_start - swa_vals = swa_start[:, None] + (offset - topk_len[:, None]) in_swa = offset < (topk_len + swa_len)[:, None] + if EXPLICIT_SWA: + swa_vals = tl.load( + swa_indices_ptr + + token.to(tl.int64)[:, None] * swa_indices_stride + + (offset - topk_len[:, None]), + mask=owned[:, None] & ~in_topk & in_swa, + other=-1, + ) + swa_vals = tl.where(swa_vals >= 0, swa_base[:, None] + swa_vals, -1) + else: + swa_start = swa_base + pos - swa_len + 1 - gather_start + swa_vals = swa_start[:, None] + (offset - topk_len[:, None]) vals = tl.where( in_topk, tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1), diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 38d33033f2..f4e89c0e50 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -232,6 +232,8 @@ def _maybe_precompute_flashmla_sched_meta( b, s_q = q.shape[0], q.shape[1] num_sm_parts = max(_num_sms(q.device.index) // s_q, 1) + if 4 * (5 * b + 1 + num_sm_parts * META_INTS) > 48 * 1024: + return meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device) num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device) flashmla_sched_meta( @@ -2382,11 +2384,15 @@ class DeepseekV4AttnBackend( ) -> SparsePrefillChunkCache: seq_lens_cpu = forward_batch.seq_lens_cpu assert seq_lens_cpu is not None - # The chunk cache gathers the W-1 positions before the chunk; under the - # tail those are late-layer window slots this prefill never wrote. - assert self.forward_metadata.late_layer_tail is None + # Decoder tails require explicit SWA indices to exclude unwritten history. + tail = self.forward_metadata.late_layer_tail + request_layout = core_attn_metadata.request_window_layout + assert tail is None or request_layout is not None extend_seq_lens = forward_batch.extend_seq_lens extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu + if tail is not None: + extend_seq_lens = tail.extend_seq_lens + extend_seq_lens_cpu = tail.extend_seq_lens_cpu assert extend_seq_lens_cpu is not None seq_lens_cpu_list = seq_lens_cpu.tolist() total_swa = sum( @@ -2409,6 +2415,9 @@ class DeepseekV4AttnBackend( query_lens = extend_seq_lens.to(torch.int32) # padding rows are never combined query_pos = core_attn_metadata.seq_lens_casual[:num_qo_tokens] - 1 + if request_layout is not None: + assert not is_cp_active(forward_batch) + query_pos = request_layout.pos.to(torch.int32) if query_pos.shape[0] < num_qo_tokens: query_pos = _pad_tensor_to_size(query_pos, num_qo_tokens, value=0) return SparsePrefillChunkCache.build( @@ -2424,6 +2433,7 @@ class DeepseekV4AttnBackend( num_qo_tokens=num_qo_tokens, max_seq_len=max(seq_lens_cpu_list), total_swa=total_swa, + request_window_layout=request_layout, ) def _build_forward_metadata( @@ -3419,13 +3429,20 @@ class DeepseekV4AttnBackend( f"{extra_indices.shape=}'s last dimension is not aligned to 64" ) - # sparse_prefill_fwd does not support SM120. The tail stays dense: its - # window floor lives in swa_page_indices, which the chunk cache ignores. + # RequestWindow sparse gathering does not support CP yet. if ( forward_batch.forward_mode.is_extend_without_speculative() and not get_platform().is_sm120 - and self.forward_metadata.late_layer_tail is None - and token_to_kv_pool.request_window is None + and ( + ( + token_to_kv_pool.request_window is None + and self.forward_metadata.late_layer_tail is None + ) + or ( + token_to_kv_pool.request_window is not None + and not is_cp_active(forward_batch) + ) + ) and ( q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() diff --git a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py index d7eb1c9af4..ba4130dc62 100644 --- a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py +++ b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py @@ -124,6 +124,8 @@ def combine_topk_swa_indices( topk: int, out_indices: Optional[torch.Tensor] = None, out_lens: Optional[torch.Tensor] = None, + swa_indices: Optional[torch.Tensor] = None, + swa_lengths: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Combine top-k and SWA indices for flash_mla_sparse_fwd. @@ -132,6 +134,7 @@ def combine_topk_swa_indices( query_start_loc can include a cross-chunk offset; query_pos is absolute. compressed_base and swa_base address the flat workspace. SWA-only layers use topk == 0, but compress_ratio must still be positive. + Optional SWA indices are relative to swa_base and preserve replay floors. Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix lengths, including -1 entries skipped by attention. Width is padded to 128. @@ -151,6 +154,10 @@ def combine_topk_swa_indices( ) num_tokens = topk_indices.shape[0] + if swa_indices is not None: + assert swa_indices.shape == (num_tokens, window_size) + assert swa_lengths is not None and swa_lengths.shape == (num_tokens,) + assert swa_indices.dtype == swa_lengths.dtype == torch.int32 num_reqs = seq_lens.shape[0] combined_topk = combined_topk_width(topk, window_size) if out_indices is None: @@ -188,9 +195,13 @@ def combine_topk_swa_indices( gather_lens, compressed_base, swa_base, + swa_indices, + 0 if swa_indices is None else swa_indices.stride(0), + swa_lengths, num_tokens, num_reqs, top_k=topk, + EXPLICIT_SWA=swa_indices is not None, COMPRESS_RATIO=compress_ratio, WINDOW_SIZE=window_size, WIDTH=combined_topk, @@ -325,6 +336,8 @@ class SparsePrefillChunkCache: # Compressed caches keyed by compress ratio: c128 (every block, combined once # per chunk) and the top-k ratios (combined per layer). compressed: Dict[int, CompressedGather] = field(default_factory=dict) + swa_indices: Optional[torch.Tensor] = None + swa_lengths: Optional[torch.Tensor] = None @classmethod def build( @@ -341,6 +354,7 @@ class SparsePrefillChunkCache: num_qo_tokens: int, max_seq_len: int, total_swa: int, + request_window_layout=None, ) -> "SparsePrefillChunkCache": """``query_lens`` / ``query_pos``: the rows this forward runs (the extend, or a CP rank's interleaved share of it); the SWA gather spans the whole extend.""" @@ -350,17 +364,29 @@ class SparsePrefillChunkCache: query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) query_start_loc[1:] = torch.cumsum(query_lens, dim=0).to(torch.int32) - swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = ( - build_swa_token_ids( - seq_lens=seq_lens, - extend_seq_lens=extend_seq_lens, - req_pool_indices=req_pool_indices, - req_to_token=req_to_token, - full_to_swa=full_to_swa, - swa_window=swa_window_size, - total_swa=total_swa, + if request_window_layout is None: + swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets = ( + build_swa_token_ids( + seq_lens=seq_lens, + extend_seq_lens=extend_seq_lens, + req_pool_indices=req_pool_indices, + req_to_token=req_to_token, + full_to_swa=full_to_swa, + swa_window=swa_window_size, + total_swa=total_swa, + ) ) - ) + swa_indices = swa_lengths = None + else: + swa_token_ids = torch.arange( + request_window_layout.size, dtype=torch.int32, device=device + ) + swa_first_pos = torch.zeros_like(seq_lens) + swa_gather_lens = torch.zeros_like(seq_lens) + # RequestWindow indices already address the shared workspace. + swa_offsets = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) + swa_indices = request_window_layout.indices + swa_lengths = request_window_layout.lengths cache = cls( num_reqs=num_reqs, @@ -375,6 +401,8 @@ class SparsePrefillChunkCache: swa_first_pos=swa_first_pos, swa_gather_lens=swa_gather_lens, swa_offsets=swa_offsets, + swa_indices=swa_indices, + swa_lengths=swa_lengths, ) # Pre-compute the c0 combine output: TOPK=0, compressed_base=0, @@ -393,6 +421,8 @@ class SparsePrefillChunkCache: window_size=swa_window_size, compress_ratio=1, topk=0, + swa_indices=swa_indices, + swa_lengths=swa_lengths, ) return cache @@ -484,6 +514,8 @@ class SparsePrefillChunkCache: window_size=self.swa_window_size, compress_ratio=128, topk=c128_max, + swa_indices=self.swa_indices, + swa_lengths=self.swa_lengths, ) gather = CompressedGather( @@ -577,4 +609,6 @@ class SparsePrefillChunkCache: topk=topk, out_indices=gather.combined_indices, out_lens=gather.combined_lens, + swa_indices=self.swa_indices, + swa_lengths=self.swa_lengths, ) diff --git a/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py b/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py index 14b37407b0..887dd37373 100644 --- a/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py +++ b/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py @@ -163,5 +163,27 @@ def test_extra_cache_schedule(b: int, topk: int, extra_topk: int): assert torch.equal(lse.view(torch.int32), ref_lse.view(torch.int32)) +def test_precompute_skips_oversized_schedule(monkeypatch): + from types import SimpleNamespace + + from sglang.srt.layers.attention import deepseek_v4_backend as backend + + monkeypatch.setattr(backend, "_fast_flashmla_sched_shape", lambda q: True) + monkeypatch.setattr(backend, "_num_sms", lambda _: 148) + max_batch = (48 * 1024 // 4 - 1 - 148 * 8) // 5 + for b in (max_batch + 1, 5000, 16384): + meta = SimpleNamespace(tile_scheduler_metadata=None, num_splits=None) + backend._maybe_precompute_flashmla_sched_meta( + meta, + q=SimpleNamespace(shape=(b, 1, 64, 512), device=SimpleNamespace(index=0)), + indices=None, + topk_length=None, + extra_indices=None, + extra_topk_length=None, + ) + assert meta.tile_scheduler_metadata is None + assert meta.num_splits is None + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py index ef87a25697..be694ad4e5 100644 --- a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py +++ b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py @@ -235,9 +235,10 @@ def _make_sparse_prefill_case( attn_sink = torch.zeros(local_heads, dtype=torch.float32, device=device) # position + 1 of the five query rows: seq_lens [96, 144], extend [3, 2] core_attn_metadata = SimpleNamespace( + request_window_layout=None, seq_lens_casual=torch.tensor( [94, 95, 96, 143, 144], dtype=torch.int32, device=device - ) + ), ) return backend, forward_batch, token_to_kv_pool, q, attn_sink, core_attn_metadata @@ -720,5 +721,140 @@ def test_q8kv8_sparse_prefill_real_kernel_repeated_launch_stable(): ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is not available") +@pytest.mark.parametrize("ratio", [0, 1, 2, 4, 128]) +@pytest.mark.parametrize("scenario", ["first", "next", "replay", "tail", "large"]) +def test_request_window_sparse_prefill_matches_paged_attention(ratio, scenario): + from sgl_kernel.flash_mla import ( + FlashMLASchedMeta, + flash_mla_with_kvcache, + ) + + from sglang.srt.mem_cache.dsv41_request_window import window_layout + from sglang.srt.runtime_context import get_platform + + if not get_platform().is_blackwell: + pytest.skip("paged FlashMLA reference requires Blackwell") + + device = torch.device("cuda") + starts = [0, 0] if scenario in ("first", "large") else [400, 900] + lens = [5000, 13] if scenario == "large" else [19, 13] + if scenario == "tail": + lens = [128, 128] + req = torch.cat([torch.full((n,), r, device=device) for r, n in zip([2, 5], lens)]) + pos = torch.cat( + [torch.arange(s, s + n, device=device) for s, n in zip(starts, lens)] + ) + floor = None + if scenario in ("replay", "tail"): + floor = torch.cat( + [torch.full((n,), s, device=device) for s, n in zip(starts, lens)] + ) + layout = window_layout(req, pos, floor=floor, num_groups=2) + seq = torch.tensor( + [s + n for s, n in zip(starts, lens)], device=device, dtype=torch.int32 + ) + lengths = torch.tensor(lens, device=device, dtype=torch.int32) + batch, _ = _make_forward_batch_and_mapping(device) + batch.seq_lens = seq + batch.seq_lens_cpu = seq.cpu() + batch.extend_seq_lens = lengths + batch.extend_seq_lens_cpu = lens + batch.req_pool_indices = torch.tensor([2, 5], device=device, dtype=torch.int32) + + page_size = 288 + swa = _make_v4_paged_kv_cache( + total_slots=layout.size, page_size=page_size, seed=13, device=device + ) + c_max = max(int(seq.max()) // max(ratio, 1), 1) + blocks = (c_max + page_size - 1) // page_size + extra = _make_v4_paged_kv_cache( + total_slots=2 * blocks * page_size, page_size=page_size, seed=17, device=device + ) + pool = _TokenToKVPool( + swa_key_buffer=swa, + full_to_swa_index_mapping=None, + page_size=page_size, + extra_key_buffer=extra, + ) + backend = _make_backend(device, None) + backend.token_to_kv_pool = pool + if scenario == "tail": + backend.forward_metadata.late_layer_tail = SimpleNamespace( + extend_seq_lens=lengths, extend_seq_lens_cpu=lens + ) + batch.extend_seq_lens = lengths + 100 + batch.extend_seq_lens_cpu = [n + 100 for n in lens] + group = torch.repeat_interleave(torch.arange(2, device=device), lengths.long()) + page_table = group[:, None] * blocks + torch.arange(blocks, device=device)[None, :] + k = ((c_max + 63) // 64) * 64 if ratio == 128 else 128 + raw = ( + torch.arange(k, device=device, dtype=torch.int32)[None, :] + .expand(pos.numel(), -1) + .contiguous() + ) + extra_lens = ((pos + 1) // max(ratio, 1)).clamp_max(k).int() + extra_ids = raw + group[:, None] * blocks * page_size + extra_ids = torch.where(raw < extra_lens[:, None], extra_ids, -1).int() + core = SimpleNamespace( + seq_lens_casual=pos.int() + 1, + request_window_layout=layout, + page_table=page_table.int(), + sparse_raw_indices=lambda _: raw, + sparse_page_indices=lambda _: extra_ids, + ) + torch.manual_seed(11) + q = torch.randn(pos.numel(), 1, 64, 512, device=device, dtype=torch.bfloat16) * 0.1 + sink = torch.zeros(64, device=device) + actual = backend._forward_prefill_sparse( + q=q, + layer_id=0, + compress_ratio=ratio, + forward_batch=batch, + token_to_kv_pool=pool, + core_attn_metadata=core, + attn_sink=sink, + ) + cache = backend.forward_metadata.sparse_prefill_cache + if ratio: + _, combined, combined_lens = cache.layer_inputs(ratio, core, page_size) + n_compressed = 2 * c_max + else: + combined, combined_lens = cache.c0_combined_indices, cache.c0_combined_lens + n_compressed = 0 + extra_lens = torch.zeros_like(extra_lens) + for i in [0, lens[0] - 1, lens[0], pos.numel() - 1]: + c = int(extra_lens[i]) + w = int(layout.lengths[i]) + assert int(combined_lens[i]) == c + w + if c: + torch.testing.assert_close(combined[i, :c], raw[i, :c] + group[i] * c_max) + torch.testing.assert_close( + combined[i, c : c + w], layout.indices[i, :w] + n_compressed + ) + kwargs = {} + if ratio: + kwargs.update( + extra_k_cache=extra.view(-1, page_size, 1, 584), + extra_indices_in_kvcache=extra_ids[:, None, :], + extra_topk_length=extra_lens, + ) + expected, _ = flash_mla_with_kvcache( + q=q, + k_cache=swa.view(-1, page_size, 1, 584), + head_dim_v=512, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=FlashMLASchedMeta(), + softmax_scale=backend.softmax_scale, + is_fp8_kvcache=True, + indices=layout.indices[:, None, :], + topk_length=layout.lengths, + attn_sink=sink, + **kwargs, + ) + torch.testing.assert_close(actual, expected.squeeze(1), atol=2e-2, rtol=2e-2) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))