[Fix] dsa: Fix ragged seed indices before MTP draft decode (#42565)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Benjamin Truong
2026-10-16 19:37:41 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 5e240a200d
commit 7c99ace127
6 changed files with 323 additions and 12 deletions
@@ -998,6 +998,39 @@ class DeepseekSparseAttnBackend(
f"Unsupported {self.dsa_topk_backend = } for SGLANG_DSA_FUSE_TOPK."
)
def prepare_mtp_seed_indices(
self, topk_indices: torch.Tensor, forward_mode: ForwardMode
) -> torch.Tensor:
"""Export a seed in the representation consumed by the next draft decode."""
if (
not self.use_fused_topk
or self.get_topk_transform_method(forward_mode)
!= TopkTransformMethod.RAGGED
):
# PAGED output is already physical; unfused/PD seeds stay request-local.
return topk_indices
# Prefill addresses concatenated logical KV, but fused draft decode
# consumes allocator-local physical slots. Leave prefill's input intact.
metadata = self.forward_metadata
page_table = metadata.page_table_1_flattened
if page_table is None:
# Without a cached prefix attention does not need a flattened table.
# Build it in the producer indexer's request order for seed export.
page_table = torch.cat(
[
metadata.page_table_1[i, :length]
for i, length in enumerate(metadata.indexer_seq_lens_cpu.tolist())
]
)
padding = topk_indices == -1
# index_select rejects invalid indices. Replace only the padding sentinel
# for the gather, then restore it; never clamp a bad positive index.
physical_indices = page_table.index_select(
0, topk_indices.masked_fill(padding, 0).reshape(-1).long()
).view_as(topk_indices)
return physical_indices.masked_fill(padding, -1)
def get_device_int32_arange(self, length: int) -> torch.Tensor:
if length > len(self._arange_buf):
next_pow_of_2 = 1 << (length - 1).bit_length()
@@ -66,13 +66,7 @@ class IndexTopKShareState:
self._forward_batch.spec_info.dsa_topk_indices = self._topk_indices
seed_buf = self._seed_buf
if seed_buf is not None:
sel = self._forward_batch.spec_info.dsa_seed_topk_select
src = (
self._topk_indices[: seed_buf.shape[0]]
if sel is None
else self._topk_indices[sel]
)
seed_buf[: src.shape[0]].copy_(src)
write_mtp_seed(self._forward_batch, seed_buf, self._topk_indices)
@classmethod
@contextmanager
@@ -95,3 +89,26 @@ class IndexTopKShareState:
finally:
spec_info.dsa_topk_indices = None
forward_batch.reuse_dsa_topk_indices = False
def write_mtp_seed(
forward_batch: ForwardBatch, seed_buf: torch.Tensor, topk_indices: torch.Tensor
) -> None:
"""Copy the selected top-k rows into the MTP seed buffer.
Both the MHA and sparse-MLA prefill paths export seeds here so the backend
can convert prefill-only index representations (e.g. fused RAGGED logical
offsets) into what the next draft decode consumes.
"""
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import (
resolve_attn_backend,
)
select = forward_batch.spec_info.dsa_seed_topk_select
src = topk_indices[: seed_buf.shape[0]] if select is None else topk_indices[select]
prepare_seed = getattr(
resolve_attn_backend(forward_batch), "prepare_mtp_seed_indices", None
)
if prepare_seed is not None:
src = prepare_seed(src, forward_batch.forward_mode)
seed_buf[: src.shape[0]].copy_(src)
@@ -7,6 +7,7 @@ import torch
from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_paged
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
from sglang.srt.environ import envs
from sglang.srt.layers.attention.index_topk_share import write_mtp_seed
from sglang.srt.layers.attention.tbo_backend import TboAttnBackend
from sglang.srt.layers.dcp import (
all_gather_kv_cache_for_mha_chunk_extend,
@@ -114,9 +115,7 @@ def forward_dsa_indexer_for_mha(
if topk_indices is None:
raise RuntimeError("DSA MHA indexer did not produce the requested MTP seed")
select = spec_info.dsa_seed_topk_select
src = topk_indices if select is None else topk_indices[select]
seed_buf[: src.shape[0]].copy_(src)
write_mtp_seed(forward_batch, seed_buf, topk_indices)
# Configs for DeepSeek-V3:
@@ -1,4 +1,4 @@
from dataclasses import dataclass
from dataclasses import dataclass, replace
from types import SimpleNamespace
from typing import Any
@@ -6,6 +6,7 @@ import torch
from torch import nn
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, ReqToTokenPool
from sglang.srt.model_executor.cuda_graph_config import (
@@ -16,6 +17,9 @@ from sglang.srt.model_executor.cuda_graph_config import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import (
forward_dsa_indexer_for_mha,
)
from sglang.srt.runtime_context import get_context, get_parallel
from .dense_attention import (
@@ -1437,6 +1441,141 @@ def run_dsa_sparse_fp8_decode_case(
)
def _make_dsa_ragged_topk(
case: DSAAttentionCase, *, index_topk: int, device: str
) -> tuple[torch.Tensor, list[list[int]]]:
"""Synthesize the fused-top-k prefill output: causal random keys per query,
addressed as batch-global offsets into the concatenated KV. Returns the
tensor and, per request, the local positions its last query selected."""
rows = []
last_rows = []
kv_start = 0
for req_idx, input_len in enumerate(case.input_lens):
for offset in range(input_len):
key_count = case.prefix_lens[req_idx] + offset + 1
local = torch.randperm(key_count)[:index_topk].sort().values.tolist()
rows.append(
[kv_start + k for k in local] + [-1] * (index_topk - len(local))
)
last_rows.append(local + [-1] * (index_topk - len(local)))
kv_start += case.seq_lens[req_idx]
return torch.tensor(rows, dtype=torch.int32, device=device), last_rows
def run_dsa_mtp_seed_handoff_case(
testcase,
case: DSAAttentionCase,
*,
seed_path: str,
index_topk: int = DSA_SPARSE_INDEX_TOPK,
device: str = DEFAULT_DEVICE,
) -> None:
"""The MTP seed a draft prefill exports must be the physical slots the first
draft decode reads. With `flashmla_sparse_q8` + fused top-k the prefill
top-k is RAGGED (batch-global logical offsets); both export paths (dense
MHA for short prefills, sparse MLA via `publish()` otherwise) must convert
it, or the decode gathers the wrong KV or reads past the pool."""
if torch.cuda.get_device_capability()[0] != 9:
testcase.skipTest("flashmla_sparse_q8 prefill is SM90-only")
if not case.forward_mode.is_extend_without_speculative():
raise ValueError("run_dsa_mtp_seed_handoff_case expects an EXTEND case.")
max_context_len = max(case.seq_lens) + case.page_size
fixture = build_dsa_sparse_attention_fixture(
testcase,
case,
max_context_len=max_context_len,
device=device,
dsa_prefill_backend="flashmla_sparse_q8",
dsa_decode_backend="flashmla_kv",
fp8_kv_cache=True,
index_topk=index_topk,
loc_layout="contiguous",
)
backend = fixture.backend
runner = fixture.runner
batch = fixture.forward_batch
testcase.assertTrue(backend.use_fused_topk)
logical, last_rows = _make_dsa_ragged_topk(
case, index_topk=index_topk, device=device
)
batch.spec_info = SimpleNamespace(
dsa_topk_indices=None,
dsa_seed_topk_capture=torch.full(
(case.batch_size, index_topk), -2, dtype=torch.int32, device=device
),
dsa_seed_topk_select=torch.tensor(case.input_lens, device=device).cumsum(0) - 1,
)
with forward_context(ForwardContext(attn_backend=backend)):
backend.init_forward_metadata(batch)
testcase.assertEqual(backend.use_mha, seed_path == "mha")
if seed_path == "mha":
forward_dsa_indexer_for_mha(
lambda **_: logical,
hidden_states=fixture.input_hidden,
q_lora=fixture.input_hidden,
positions=batch.positions,
forward_batch=batch,
layer_id=0,
)
else:
state = IndexTopKShareState.from_mtp_carry(batch)
state.update(logical)
state.publish()
seed = batch.spec_info.dsa_seed_topk_capture
local_rows = torch.tensor(last_rows, device=device)
req_to_token = runner.req_to_token_pool.req_to_token[: case.batch_size]
expected_seed = torch.where(
local_rows >= 0, req_to_token.gather(1, local_rows.clamp(min=0).long()), -1
).to(torch.int32)
torch.testing.assert_close(seed, expected_seed, rtol=0, atol=0)
# First draft decode after that prefill, carrying the seed.
decode_case = DSAAttentionCase(
name=f"{case.name}_decode",
backend=case.backend,
forward_mode=ForwardMode.DECODE,
num_heads=case.num_heads,
num_kv_heads=case.num_kv_heads,
page_size=case.page_size,
prefix_lens=case.seq_lens,
)
decode_batch = _make_forward_batch(
decode_case, runner, max_context_len=max_context_len, device=device
)
decode_batch.spec_info = SimpleNamespace(
dsa_topk_indices=seed, dsa_seed_topk_capture=None, dsa_seed_topk_select=None
)
input_parts = _split_by_lens(fixture.input_hidden, case.input_lens)
decode = replace(
fixture,
case=decode_case,
forward_batch=decode_batch,
prefix_hidden=[
torch.cat([prefix, part])
for prefix, part in zip(fixture.prefix_hidden, input_parts)
],
input_hidden=torch.randn_like(fixture.input_hidden[: case.batch_size]),
topk_rows=last_rows,
)
_populate_dsa_sparse_prefix_kv(
fixture.actual_module,
decode_case,
runner,
decode.prefix_hidden,
max_context_len=max_context_len,
)
with IndexTopKShareState.mtp_iteration(decode_batch, keep_carry_seed=True) as state:
decode.topk_indices = state.topk_indices
actual = run_dsa_sparse_fixture_eager(decode, testcase)
torch.testing.assert_close(
actual,
expected_dsa_sparse_fixture_output(decode),
atol=DSA_SPARSE_FP8_ATOL,
rtol=DSA_SPARSE_FP8_RTOL,
)
def run_dsa_sparse_cuda_graph_decode_impl_variant_case(
testcase,
case: DSAAttentionCase,