[DeepSeek V4.1] Use DeepSelect for Hopper full top-k decode. (#41561)

Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
yuyu5333
2026-10-14 10:46:12 +08:00
committed by GitHub
co-authored by Claude Code
parent 87beb3a052
commit b50fdeb0ef
3 changed files with 76 additions and 1 deletions
@@ -7,6 +7,10 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.ops.attention.deep_select import (
is_deep_select_supported,
topk_page_transform,
)
from sglang.kernels.ops.attention.dsv4.index_logits import (
deep_gemm_fp4_paged_mqa_logits,
)
@@ -47,6 +51,9 @@ class FullTopKIndexer:
self.req_to_token = req_to_token
self.use_deep_gemm_prefill = use_deep_gemm_prefill
self.use_deep_gemm_decode = use_deep_gemm_decode
self.use_deep_select_decode = (
not use_deep_gemm_decode and is_deep_select_supported()
)
def topk_prefill(self, inputs: PrefillInputs) -> None:
if self.use_deep_gemm_prefill:
@@ -134,6 +141,18 @@ class FullTopKIndexer:
)
if d is None:
return
if self.use_deep_select_decode:
metadata = inputs.paged_metadata
topk_page_transform(
d.scores,
inputs.indexer.index_topk,
page_table=metadata.page_table[: d.bs],
page_size=metadata.compressed_page_size,
end=d.lens.to(torch.int32),
sorted_index=False,
output_idx=inputs.out_page_indices[: d.bs],
)
return
select_decode(inputs, d, inputs.indexer.index_topk)
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
@@ -148,6 +148,7 @@ def _random_page_table(rows: int, width: int, page_size: int):
@pytest.mark.parametrize(
"k,rows,width,with_end",
[
(512, 4, 64, True),
(2048, 150, 33000, True),
(1024, 2, 600_000, False),
],
@@ -161,7 +162,7 @@ def test_page_transform_matches_reference(k, rows, width, with_end):
end = torch.randint(
1, width + 1, (rows,), dtype=torch.int32, device="cuda", generator=g
)
end[0] = k // 2
end[0] = min(k // 2, width)
table, inverse = _random_page_table(rows, width, page_size)
slots = topk_page_transform(
@@ -904,6 +904,61 @@ class TestChunkedTopKMatchesUnchunked(CustomTestCase):
self.assertTrue(torch.equal(run(rows_per_chunk), expected))
class TestDeepSelectFullTopKDecode(CustomTestCase):
@staticmethod
def _decode_case():
from sglang.srt.layers.attention.dsv4.v41_indexer import DecodeInputs
from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import DecodeScores
scores = torch.tensor(
[[1.0, 4.0, 2.0, 3.0], [3.0, 2.0, -torch.inf, -torch.inf]],
dtype=torch.float32,
)
lens = torch.tensor([4, 2], dtype=torch.int64)
slots = torch.tensor([[40, 10, 30, 20], [80, 70, 0, 0]], dtype=torch.int64)
metadata = SimpleNamespace(
page_table=torch.tensor([[10], [20]], dtype=torch.int32),
compressed_page_size=4,
)
page_indices = torch.full((2, 8), -1, dtype=torch.int32)
inputs = DecodeInputs(
indexer=SimpleNamespace(index_topk=8),
layer_id=0,
compress_ratio=1,
freqs_cis=None,
x=None,
q_lora=None,
positions=None,
req_rows=torch.arange(2),
paged_metadata=metadata,
is_verify=False,
out_page_indices=page_indices,
)
data = DecodeScores(bs=2, lmax=4, lens=lens, slots=slots, scores=scores)
return inputs, data
def test_decode_uses_select_decode_fallback(self):
from sglang.srt.layers.attention.dsv4.v41_indexer import full_topk as mod
inputs, data = self._decode_case()
inputs.indexer.index_topk = 2
indexer = object.__new__(mod.FullTopKIndexer)
indexer.token_to_kv_pool = None
indexer.req_to_token = None
indexer.use_deep_gemm_decode = False
indexer.use_deep_select_decode = False
with (
patch.object(mod, "decode_scores", return_value=data),
patch.object(mod, "topk_page_transform") as page_transform,
patch.object(mod, "select_decode") as select_decode,
):
indexer.topk_decode(inputs)
page_transform.assert_not_called()
select_decode.assert_called_once_with(inputs, data, 2)
class TestChunkedPagedDecode(CustomTestCase):
"""An eager decode whose paged-MQA metadata is split into row chunks (#40637):
every DeepGEMM call must get its own chunk's schedule and every top-k its own