[Fix] dsa: Fix ragged seed indices before MTP draft decode (#42565)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
5e240a200d
commit
7c99ace127
@@ -998,6 +998,39 @@ class DeepseekSparseAttnBackend(
|
||||
f"Unsupported {self.dsa_topk_backend = } for SGLANG_DSA_FUSE_TOPK."
|
||||
)
|
||||
|
||||
def prepare_mtp_seed_indices(
|
||||
self, topk_indices: torch.Tensor, forward_mode: ForwardMode
|
||||
) -> torch.Tensor:
|
||||
"""Export a seed in the representation consumed by the next draft decode."""
|
||||
if (
|
||||
not self.use_fused_topk
|
||||
or self.get_topk_transform_method(forward_mode)
|
||||
!= TopkTransformMethod.RAGGED
|
||||
):
|
||||
# PAGED output is already physical; unfused/PD seeds stay request-local.
|
||||
return topk_indices
|
||||
|
||||
# Prefill addresses concatenated logical KV, but fused draft decode
|
||||
# consumes allocator-local physical slots. Leave prefill's input intact.
|
||||
metadata = self.forward_metadata
|
||||
page_table = metadata.page_table_1_flattened
|
||||
if page_table is None:
|
||||
# Without a cached prefix attention does not need a flattened table.
|
||||
# Build it in the producer indexer's request order for seed export.
|
||||
page_table = torch.cat(
|
||||
[
|
||||
metadata.page_table_1[i, :length]
|
||||
for i, length in enumerate(metadata.indexer_seq_lens_cpu.tolist())
|
||||
]
|
||||
)
|
||||
padding = topk_indices == -1
|
||||
# index_select rejects invalid indices. Replace only the padding sentinel
|
||||
# for the gather, then restore it; never clamp a bad positive index.
|
||||
physical_indices = page_table.index_select(
|
||||
0, topk_indices.masked_fill(padding, 0).reshape(-1).long()
|
||||
).view_as(topk_indices)
|
||||
return physical_indices.masked_fill(padding, -1)
|
||||
|
||||
def get_device_int32_arange(self, length: int) -> torch.Tensor:
|
||||
if length > len(self._arange_buf):
|
||||
next_pow_of_2 = 1 << (length - 1).bit_length()
|
||||
|
||||
@@ -66,13 +66,7 @@ class IndexTopKShareState:
|
||||
self._forward_batch.spec_info.dsa_topk_indices = self._topk_indices
|
||||
seed_buf = self._seed_buf
|
||||
if seed_buf is not None:
|
||||
sel = self._forward_batch.spec_info.dsa_seed_topk_select
|
||||
src = (
|
||||
self._topk_indices[: seed_buf.shape[0]]
|
||||
if sel is None
|
||||
else self._topk_indices[sel]
|
||||
)
|
||||
seed_buf[: src.shape[0]].copy_(src)
|
||||
write_mtp_seed(self._forward_batch, seed_buf, self._topk_indices)
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
@@ -95,3 +89,26 @@ class IndexTopKShareState:
|
||||
finally:
|
||||
spec_info.dsa_topk_indices = None
|
||||
forward_batch.reuse_dsa_topk_indices = False
|
||||
|
||||
|
||||
def write_mtp_seed(
|
||||
forward_batch: ForwardBatch, seed_buf: torch.Tensor, topk_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Copy the selected top-k rows into the MTP seed buffer.
|
||||
|
||||
Both the MHA and sparse-MLA prefill paths export seeds here so the backend
|
||||
can convert prefill-only index representations (e.g. fused RAGGED logical
|
||||
offsets) into what the next draft decode consumes.
|
||||
"""
|
||||
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import (
|
||||
resolve_attn_backend,
|
||||
)
|
||||
|
||||
select = forward_batch.spec_info.dsa_seed_topk_select
|
||||
src = topk_indices[: seed_buf.shape[0]] if select is None else topk_indices[select]
|
||||
prepare_seed = getattr(
|
||||
resolve_attn_backend(forward_batch), "prepare_mtp_seed_indices", None
|
||||
)
|
||||
if prepare_seed is not None:
|
||||
src = prepare_seed(src, forward_batch.forward_mode)
|
||||
seed_buf[: src.shape[0]].copy_(src)
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_paged
|
||||
from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.index_topk_share import write_mtp_seed
|
||||
from sglang.srt.layers.attention.tbo_backend import TboAttnBackend
|
||||
from sglang.srt.layers.dcp import (
|
||||
all_gather_kv_cache_for_mha_chunk_extend,
|
||||
@@ -114,9 +115,7 @@ def forward_dsa_indexer_for_mha(
|
||||
if topk_indices is None:
|
||||
raise RuntimeError("DSA MHA indexer did not produce the requested MTP seed")
|
||||
|
||||
select = spec_info.dsa_seed_topk_select
|
||||
src = topk_indices if select is None else topk_indices[select]
|
||||
seed_buf[: src.shape[0]].copy_(src)
|
||||
write_mtp_seed(forward_batch, seed_buf, topk_indices)
|
||||
|
||||
|
||||
# Configs for DeepSeek-V3:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, ReqToTokenPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -16,6 +17,9 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import (
|
||||
forward_dsa_indexer_for_mha,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
|
||||
from .dense_attention import (
|
||||
@@ -1437,6 +1441,141 @@ def run_dsa_sparse_fp8_decode_case(
|
||||
)
|
||||
|
||||
|
||||
def _make_dsa_ragged_topk(
|
||||
case: DSAAttentionCase, *, index_topk: int, device: str
|
||||
) -> tuple[torch.Tensor, list[list[int]]]:
|
||||
"""Synthesize the fused-top-k prefill output: causal random keys per query,
|
||||
addressed as batch-global offsets into the concatenated KV. Returns the
|
||||
tensor and, per request, the local positions its last query selected."""
|
||||
rows = []
|
||||
last_rows = []
|
||||
kv_start = 0
|
||||
for req_idx, input_len in enumerate(case.input_lens):
|
||||
for offset in range(input_len):
|
||||
key_count = case.prefix_lens[req_idx] + offset + 1
|
||||
local = torch.randperm(key_count)[:index_topk].sort().values.tolist()
|
||||
rows.append(
|
||||
[kv_start + k for k in local] + [-1] * (index_topk - len(local))
|
||||
)
|
||||
last_rows.append(local + [-1] * (index_topk - len(local)))
|
||||
kv_start += case.seq_lens[req_idx]
|
||||
return torch.tensor(rows, dtype=torch.int32, device=device), last_rows
|
||||
|
||||
|
||||
def run_dsa_mtp_seed_handoff_case(
|
||||
testcase,
|
||||
case: DSAAttentionCase,
|
||||
*,
|
||||
seed_path: str,
|
||||
index_topk: int = DSA_SPARSE_INDEX_TOPK,
|
||||
device: str = DEFAULT_DEVICE,
|
||||
) -> None:
|
||||
"""The MTP seed a draft prefill exports must be the physical slots the first
|
||||
draft decode reads. With `flashmla_sparse_q8` + fused top-k the prefill
|
||||
top-k is RAGGED (batch-global logical offsets); both export paths (dense
|
||||
MHA for short prefills, sparse MLA via `publish()` otherwise) must convert
|
||||
it, or the decode gathers the wrong KV or reads past the pool."""
|
||||
if torch.cuda.get_device_capability()[0] != 9:
|
||||
testcase.skipTest("flashmla_sparse_q8 prefill is SM90-only")
|
||||
if not case.forward_mode.is_extend_without_speculative():
|
||||
raise ValueError("run_dsa_mtp_seed_handoff_case expects an EXTEND case.")
|
||||
max_context_len = max(case.seq_lens) + case.page_size
|
||||
fixture = build_dsa_sparse_attention_fixture(
|
||||
testcase,
|
||||
case,
|
||||
max_context_len=max_context_len,
|
||||
device=device,
|
||||
dsa_prefill_backend="flashmla_sparse_q8",
|
||||
dsa_decode_backend="flashmla_kv",
|
||||
fp8_kv_cache=True,
|
||||
index_topk=index_topk,
|
||||
loc_layout="contiguous",
|
||||
)
|
||||
backend = fixture.backend
|
||||
runner = fixture.runner
|
||||
batch = fixture.forward_batch
|
||||
testcase.assertTrue(backend.use_fused_topk)
|
||||
logical, last_rows = _make_dsa_ragged_topk(
|
||||
case, index_topk=index_topk, device=device
|
||||
)
|
||||
batch.spec_info = SimpleNamespace(
|
||||
dsa_topk_indices=None,
|
||||
dsa_seed_topk_capture=torch.full(
|
||||
(case.batch_size, index_topk), -2, dtype=torch.int32, device=device
|
||||
),
|
||||
dsa_seed_topk_select=torch.tensor(case.input_lens, device=device).cumsum(0) - 1,
|
||||
)
|
||||
with forward_context(ForwardContext(attn_backend=backend)):
|
||||
backend.init_forward_metadata(batch)
|
||||
testcase.assertEqual(backend.use_mha, seed_path == "mha")
|
||||
if seed_path == "mha":
|
||||
forward_dsa_indexer_for_mha(
|
||||
lambda **_: logical,
|
||||
hidden_states=fixture.input_hidden,
|
||||
q_lora=fixture.input_hidden,
|
||||
positions=batch.positions,
|
||||
forward_batch=batch,
|
||||
layer_id=0,
|
||||
)
|
||||
else:
|
||||
state = IndexTopKShareState.from_mtp_carry(batch)
|
||||
state.update(logical)
|
||||
state.publish()
|
||||
seed = batch.spec_info.dsa_seed_topk_capture
|
||||
|
||||
local_rows = torch.tensor(last_rows, device=device)
|
||||
req_to_token = runner.req_to_token_pool.req_to_token[: case.batch_size]
|
||||
expected_seed = torch.where(
|
||||
local_rows >= 0, req_to_token.gather(1, local_rows.clamp(min=0).long()), -1
|
||||
).to(torch.int32)
|
||||
torch.testing.assert_close(seed, expected_seed, rtol=0, atol=0)
|
||||
|
||||
# First draft decode after that prefill, carrying the seed.
|
||||
decode_case = DSAAttentionCase(
|
||||
name=f"{case.name}_decode",
|
||||
backend=case.backend,
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
num_heads=case.num_heads,
|
||||
num_kv_heads=case.num_kv_heads,
|
||||
page_size=case.page_size,
|
||||
prefix_lens=case.seq_lens,
|
||||
)
|
||||
decode_batch = _make_forward_batch(
|
||||
decode_case, runner, max_context_len=max_context_len, device=device
|
||||
)
|
||||
decode_batch.spec_info = SimpleNamespace(
|
||||
dsa_topk_indices=seed, dsa_seed_topk_capture=None, dsa_seed_topk_select=None
|
||||
)
|
||||
input_parts = _split_by_lens(fixture.input_hidden, case.input_lens)
|
||||
decode = replace(
|
||||
fixture,
|
||||
case=decode_case,
|
||||
forward_batch=decode_batch,
|
||||
prefix_hidden=[
|
||||
torch.cat([prefix, part])
|
||||
for prefix, part in zip(fixture.prefix_hidden, input_parts)
|
||||
],
|
||||
input_hidden=torch.randn_like(fixture.input_hidden[: case.batch_size]),
|
||||
topk_rows=last_rows,
|
||||
)
|
||||
_populate_dsa_sparse_prefix_kv(
|
||||
fixture.actual_module,
|
||||
decode_case,
|
||||
runner,
|
||||
decode.prefix_hidden,
|
||||
max_context_len=max_context_len,
|
||||
)
|
||||
with IndexTopKShareState.mtp_iteration(decode_batch, keep_carry_seed=True) as state:
|
||||
decode.topk_indices = state.topk_indices
|
||||
actual = run_dsa_sparse_fixture_eager(decode, testcase)
|
||||
torch.testing.assert_close(
|
||||
actual,
|
||||
expected_dsa_sparse_fixture_output(decode),
|
||||
atol=DSA_SPARSE_FP8_ATOL,
|
||||
rtol=DSA_SPARSE_FP8_RTOL,
|
||||
)
|
||||
|
||||
|
||||
def run_dsa_sparse_cuda_graph_decode_impl_variant_case(
|
||||
testcase,
|
||||
case: DSAAttentionCase,
|
||||
|
||||
Reference in New Issue
Block a user