From 7c99ace127526c2a7b21e5474a2981e4789b367c Mon Sep 17 00:00:00 2001 From: Benjamin Truong Date: Fri, 16 Oct 2026 18:37:41 +0700 Subject: [PATCH] [Fix] dsa: Fix ragged seed indices before MTP draft decode (#42565) Co-authored-by: Claude Opus 5.5 --- .../srt/layers/attention/dsa_backend.py | 33 ++++ .../srt/layers/attention/index_topk_share.py | 31 +++- .../attention_forward_methods/forward_mha.py | 5 +- .../attention_methods/dsa_attention.py | 141 +++++++++++++++++- .../attention/unittests/dsa/test_dsa.py | 39 +++++ .../layers/attention/test_index_topk_share.py | 86 ++++++++++- 6 files changed, 323 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 948e9ee742..ae1de8455e 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -998,6 +998,39 @@ class DeepseekSparseAttnBackend( f"Unsupported {self.dsa_topk_backend = } for SGLANG_DSA_FUSE_TOPK." ) + def prepare_mtp_seed_indices( + self, topk_indices: torch.Tensor, forward_mode: ForwardMode + ) -> torch.Tensor: + """Export a seed in the representation consumed by the next draft decode.""" + if ( + not self.use_fused_topk + or self.get_topk_transform_method(forward_mode) + != TopkTransformMethod.RAGGED + ): + # PAGED output is already physical; unfused/PD seeds stay request-local. + return topk_indices + + # Prefill addresses concatenated logical KV, but fused draft decode + # consumes allocator-local physical slots. Leave prefill's input intact. + metadata = self.forward_metadata + page_table = metadata.page_table_1_flattened + if page_table is None: + # Without a cached prefix attention does not need a flattened table. + # Build it in the producer indexer's request order for seed export. + page_table = torch.cat( + [ + metadata.page_table_1[i, :length] + for i, length in enumerate(metadata.indexer_seq_lens_cpu.tolist()) + ] + ) + padding = topk_indices == -1 + # index_select rejects invalid indices. Replace only the padding sentinel + # for the gather, then restore it; never clamp a bad positive index. + physical_indices = page_table.index_select( + 0, topk_indices.masked_fill(padding, 0).reshape(-1).long() + ).view_as(topk_indices) + return physical_indices.masked_fill(padding, -1) + def get_device_int32_arange(self, length: int) -> torch.Tensor: if length > len(self._arange_buf): next_pow_of_2 = 1 << (length - 1).bit_length() diff --git a/python/sglang/srt/layers/attention/index_topk_share.py b/python/sglang/srt/layers/attention/index_topk_share.py index 7b56b487cf..a45c716ebe 100644 --- a/python/sglang/srt/layers/attention/index_topk_share.py +++ b/python/sglang/srt/layers/attention/index_topk_share.py @@ -66,13 +66,7 @@ class IndexTopKShareState: self._forward_batch.spec_info.dsa_topk_indices = self._topk_indices seed_buf = self._seed_buf if seed_buf is not None: - sel = self._forward_batch.spec_info.dsa_seed_topk_select - src = ( - self._topk_indices[: seed_buf.shape[0]] - if sel is None - else self._topk_indices[sel] - ) - seed_buf[: src.shape[0]].copy_(src) + write_mtp_seed(self._forward_batch, seed_buf, self._topk_indices) @classmethod @contextmanager @@ -95,3 +89,26 @@ class IndexTopKShareState: finally: spec_info.dsa_topk_indices = None forward_batch.reuse_dsa_topk_indices = False + + +def write_mtp_seed( + forward_batch: ForwardBatch, seed_buf: torch.Tensor, topk_indices: torch.Tensor +) -> None: + """Copy the selected top-k rows into the MTP seed buffer. + + Both the MHA and sparse-MLA prefill paths export seeds here so the backend + can convert prefill-only index representations (e.g. fused RAGGED logical + offsets) into what the next draft decode consumes. + """ + from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import ( + resolve_attn_backend, + ) + + select = forward_batch.spec_info.dsa_seed_topk_select + src = topk_indices[: seed_buf.shape[0]] if select is None else topk_indices[select] + prepare_seed = getattr( + resolve_attn_backend(forward_batch), "prepare_mtp_seed_indices", None + ) + if prepare_seed is not None: + src = prepare_seed(src, forward_batch.forward_mode) + seed_buf[: src.shape[0]].copy_(src) diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index a625079f51..0ede89212c 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -7,6 +7,7 @@ import torch from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_paged from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton from sglang.srt.environ import envs +from sglang.srt.layers.attention.index_topk_share import write_mtp_seed from sglang.srt.layers.attention.tbo_backend import TboAttnBackend from sglang.srt.layers.dcp import ( all_gather_kv_cache_for_mha_chunk_extend, @@ -114,9 +115,7 @@ def forward_dsa_indexer_for_mha( if topk_indices is None: raise RuntimeError("DSA MHA indexer did not produce the requested MTP seed") - select = spec_info.dsa_seed_topk_select - src = topk_indices if select is None else topk_indices[select] - seed_buf[: src.shape[0]].copy_(src) + write_mtp_seed(forward_batch, seed_buf, topk_indices) # Configs for DeepSeek-V3: diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index faebbeaeed..67bbc566d8 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, replace from types import SimpleNamespace from typing import Any @@ -6,6 +6,7 @@ import torch from torch import nn from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS +from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, ReqToTokenPool from sglang.srt.model_executor.cuda_graph_config import ( @@ -16,6 +17,9 @@ from sglang.srt.model_executor.cuda_graph_config import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import ( + forward_dsa_indexer_for_mha, +) from sglang.srt.runtime_context import get_context, get_parallel from .dense_attention import ( @@ -1437,6 +1441,141 @@ def run_dsa_sparse_fp8_decode_case( ) +def _make_dsa_ragged_topk( + case: DSAAttentionCase, *, index_topk: int, device: str +) -> tuple[torch.Tensor, list[list[int]]]: + """Synthesize the fused-top-k prefill output: causal random keys per query, + addressed as batch-global offsets into the concatenated KV. Returns the + tensor and, per request, the local positions its last query selected.""" + rows = [] + last_rows = [] + kv_start = 0 + for req_idx, input_len in enumerate(case.input_lens): + for offset in range(input_len): + key_count = case.prefix_lens[req_idx] + offset + 1 + local = torch.randperm(key_count)[:index_topk].sort().values.tolist() + rows.append( + [kv_start + k for k in local] + [-1] * (index_topk - len(local)) + ) + last_rows.append(local + [-1] * (index_topk - len(local))) + kv_start += case.seq_lens[req_idx] + return torch.tensor(rows, dtype=torch.int32, device=device), last_rows + + +def run_dsa_mtp_seed_handoff_case( + testcase, + case: DSAAttentionCase, + *, + seed_path: str, + index_topk: int = DSA_SPARSE_INDEX_TOPK, + device: str = DEFAULT_DEVICE, +) -> None: + """The MTP seed a draft prefill exports must be the physical slots the first + draft decode reads. With `flashmla_sparse_q8` + fused top-k the prefill + top-k is RAGGED (batch-global logical offsets); both export paths (dense + MHA for short prefills, sparse MLA via `publish()` otherwise) must convert + it, or the decode gathers the wrong KV or reads past the pool.""" + if torch.cuda.get_device_capability()[0] != 9: + testcase.skipTest("flashmla_sparse_q8 prefill is SM90-only") + if not case.forward_mode.is_extend_without_speculative(): + raise ValueError("run_dsa_mtp_seed_handoff_case expects an EXTEND case.") + max_context_len = max(case.seq_lens) + case.page_size + fixture = build_dsa_sparse_attention_fixture( + testcase, + case, + max_context_len=max_context_len, + device=device, + dsa_prefill_backend="flashmla_sparse_q8", + dsa_decode_backend="flashmla_kv", + fp8_kv_cache=True, + index_topk=index_topk, + loc_layout="contiguous", + ) + backend = fixture.backend + runner = fixture.runner + batch = fixture.forward_batch + testcase.assertTrue(backend.use_fused_topk) + logical, last_rows = _make_dsa_ragged_topk( + case, index_topk=index_topk, device=device + ) + batch.spec_info = SimpleNamespace( + dsa_topk_indices=None, + dsa_seed_topk_capture=torch.full( + (case.batch_size, index_topk), -2, dtype=torch.int32, device=device + ), + dsa_seed_topk_select=torch.tensor(case.input_lens, device=device).cumsum(0) - 1, + ) + with forward_context(ForwardContext(attn_backend=backend)): + backend.init_forward_metadata(batch) + testcase.assertEqual(backend.use_mha, seed_path == "mha") + if seed_path == "mha": + forward_dsa_indexer_for_mha( + lambda **_: logical, + hidden_states=fixture.input_hidden, + q_lora=fixture.input_hidden, + positions=batch.positions, + forward_batch=batch, + layer_id=0, + ) + else: + state = IndexTopKShareState.from_mtp_carry(batch) + state.update(logical) + state.publish() + seed = batch.spec_info.dsa_seed_topk_capture + + local_rows = torch.tensor(last_rows, device=device) + req_to_token = runner.req_to_token_pool.req_to_token[: case.batch_size] + expected_seed = torch.where( + local_rows >= 0, req_to_token.gather(1, local_rows.clamp(min=0).long()), -1 + ).to(torch.int32) + torch.testing.assert_close(seed, expected_seed, rtol=0, atol=0) + + # First draft decode after that prefill, carrying the seed. + decode_case = DSAAttentionCase( + name=f"{case.name}_decode", + backend=case.backend, + forward_mode=ForwardMode.DECODE, + num_heads=case.num_heads, + num_kv_heads=case.num_kv_heads, + page_size=case.page_size, + prefix_lens=case.seq_lens, + ) + decode_batch = _make_forward_batch( + decode_case, runner, max_context_len=max_context_len, device=device + ) + decode_batch.spec_info = SimpleNamespace( + dsa_topk_indices=seed, dsa_seed_topk_capture=None, dsa_seed_topk_select=None + ) + input_parts = _split_by_lens(fixture.input_hidden, case.input_lens) + decode = replace( + fixture, + case=decode_case, + forward_batch=decode_batch, + prefix_hidden=[ + torch.cat([prefix, part]) + for prefix, part in zip(fixture.prefix_hidden, input_parts) + ], + input_hidden=torch.randn_like(fixture.input_hidden[: case.batch_size]), + topk_rows=last_rows, + ) + _populate_dsa_sparse_prefix_kv( + fixture.actual_module, + decode_case, + runner, + decode.prefix_hidden, + max_context_len=max_context_len, + ) + with IndexTopKShareState.mtp_iteration(decode_batch, keep_carry_seed=True) as state: + decode.topk_indices = state.topk_indices + actual = run_dsa_sparse_fixture_eager(decode, testcase) + torch.testing.assert_close( + actual, + expected_dsa_sparse_fixture_output(decode), + atol=DSA_SPARSE_FP8_ATOL, + rtol=DSA_SPARSE_FP8_RTOL, + ) + + def run_dsa_sparse_cuda_graph_decode_impl_variant_case( testcase, case: DSAAttentionCase, diff --git a/test/registered/attention/unittests/dsa/test_dsa.py b/test/registered/attention/unittests/dsa/test_dsa.py index 3c63513ce8..2d88f2d388 100644 --- a/test/registered/attention/unittests/dsa/test_dsa.py +++ b/test/registered/attention/unittests/dsa/test_dsa.py @@ -12,6 +12,7 @@ from sglang.test.kits.attention_unittest.attention_methods.dsa_attention import make_dsa_dense_fallback_cases, make_dsa_sparse_cases, run_dsa_attention_case, + run_dsa_mtp_seed_handoff_case, run_dsa_sparse_attention_case, run_dsa_sparse_cuda_graph_decode_impl_variant_case, run_dsa_sparse_decode_impl_variant_case, @@ -395,6 +396,44 @@ class TestDSAAttentionBackendCorrectness(CustomTestCase): self, self.CUDA_GRAPH_DECODE_CASES[0], impl ) + # MTP seed handoff with fused top-k + fp8 KV + `flashmla_sparse_q8`: the + # draft prefill's RAGGED top-k must reach the first draft decode as + # physical slots. Short prefills export through dense MHA, longer ones + # through the sparse-MLA `publish()` path; both must convert. + MTP_SEED_HANDOFF_CASES = ( + ( + DSAAttentionCase( + name="dsa_mtp_seed_handoff_mha", + backend="dsa", + forward_mode=ForwardMode.EXTEND, + num_heads=4, + num_kv_heads=1, + page_size=DSA_PAGE_SIZE, + prefix_lens=(512, 256), + extend_lens=(3, 2), + ), + "mha", + ), + ( + DSAAttentionCase( + name="dsa_mtp_seed_handoff_sparse_mla", + backend="dsa", + forward_mode=ForwardMode.EXTEND, + num_heads=4, + num_kv_heads=1, + page_size=DSA_PAGE_SIZE, + prefix_lens=(2048, 1024), + extend_lens=(3, 2), + ), + "sparse_mla", + ), + ) + + def test_mtp_seed_handoff_cases(self): + for case, seed_path in self.MTP_SEED_HANDOFF_CASES: + with self.subTest(case=case.name, seed_path=seed_path): + run_dsa_mtp_seed_handoff_case(self, case, seed_path=seed_path) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/attention/test_index_topk_share.py b/test/registered/unit/layers/attention/test_index_topk_share.py index f92a2bb340..88c49a6e86 100644 --- a/test/registered/unit/layers/attention/test_index_topk_share.py +++ b/test/registered/unit/layers/attention/test_index_topk_share.py @@ -1,11 +1,15 @@ import sys from types import SimpleNamespace +from unittest.mock import Mock, patch import pytest import torch +from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod +from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.models.deepseek_common.attention_forward_methods import forward_mha from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=6, suite="base-a-test-cpu") @@ -81,7 +85,8 @@ def test_publish_captures_draft_extend_seed(): assert state.should_publish state.update(torch.arange(12, dtype=torch.int64).view(4, 3)) - state.publish() + with patch.object(forward_mha, "resolve_attn_backend", return_value=None): + state.publish() assert torch.equal(seed_buf, torch.arange(6, dtype=torch.int64).view(2, 3)) assert batch.spec_info.dsa_topk_indices is None @@ -158,5 +163,84 @@ def test_mtp_iteration_clears_missing_draft_extend_seed(): assert batch.spec_info.dsa_topk_indices is None +def _capture( + indices, *, fused=True, ragged=True, flattened=True, select=None, path="mha" +): + # Two requests share their first two physical slots. Logical index 5 is + # outside the four-slot physical pool, but maps to the valid physical slot 2. + table = torch.tensor([[3, 1, 0], [3, 1, 2]], dtype=torch.int32) + metadata = SimpleNamespace( + page_table_1=table, + page_table_1_flattened=table.flatten() if flattened else None, + indexer_seq_lens_cpu=torch.tensor([3, 3]), + ) + # Exercise the real seed export method without initializing GPU resources. + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend.use_fused_topk = fused + backend.get_topk_transform_method = Mock( + return_value=TopkTransformMethod.RAGGED if ragged else TopkTransformMethod.PAGED + ) + backend.forward_metadata = metadata + rows = len(indices) if select is None else len(select) + capture = torch.full((rows + 1, indices.shape[1]), -99, dtype=torch.int32) + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + reuse_dsa_topk_indices=False, + spec_info=SimpleNamespace( + dsa_seed_topk_capture=capture, + dsa_seed_topk_select=select, + ), + ) + with patch.object(forward_mha, "resolve_attn_backend", return_value=backend): + if path == "mha": + indexer = Mock(return_value=indices) + forward_mha.forward_dsa_indexer_for_mha( + indexer, + hidden_states=None, + q_lora=None, + positions=None, + forward_batch=batch, + layer_id=0, + ) + assert indexer.call_args.kwargs["return_indices"] + else: + # Long prefills take sparse MLA and export through publish(). + state = IndexTopKShareState.from_mtp_carry(batch) + state.update(indices) + state.publish() + assert torch.all(capture[-1] == -99) + return capture[:-1] + + +@pytest.mark.parametrize("path", ["mha", "publish"]) +@pytest.mark.parametrize("flattened", [True, False]) +@pytest.mark.parametrize("select", [None, torch.tensor([2, 0])]) +def test_ragged_seed_maps_shared_prefix_and_preserves_padding(flattened, select, path): + """Export physical slots, even when shared prefixes make ragged offsets larger.""" + indices = torch.tensor([[0, 2, -1], [3, 4, -1], [5, 1, -1]], dtype=torch.int32) + original = indices.clone() + expected = torch.tensor([[3, 0, -1], [3, 1, -1], [2, 1, -1]], dtype=torch.int32) + if select is not None: + expected = expected[select] + actual = _capture(indices, flattened=flattened, select=select, path=path) + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(indices, original) + + +@pytest.mark.parametrize("path", ["mha", "publish"]) +@pytest.mark.parametrize("fused,ragged", [(True, False), (False, True), (False, False)]) +def test_paged_and_unfused_seed_contracts_are_preserved(fused, ragged, path): + indices = torch.tensor([[2, 0, -1], [1, 2, -1]], dtype=torch.int32) + actual = _capture(indices, fused=fused, ragged=ragged, path=path) + torch.testing.assert_close(actual, indices) + + +@pytest.mark.parametrize("path", ["mha", "publish"]) +@pytest.mark.parametrize("invalid", [-2, 6]) +def test_invalid_ragged_indices_are_not_clamped(invalid, path): + with pytest.raises((IndexError, RuntimeError)): + _capture(torch.tensor([[invalid, -1]], dtype=torch.int32), path=path) + + if __name__ == "__main__": sys.exit(pytest.main([__file__]))