[DeepSeek V4.1] Use DeepSelect for Hopper full top-k decode. (#41561)
Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.deep_select import (
|
||||
is_deep_select_supported,
|
||||
topk_page_transform,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import (
|
||||
deep_gemm_fp4_paged_mqa_logits,
|
||||
)
|
||||
@@ -47,6 +51,9 @@ class FullTopKIndexer:
|
||||
self.req_to_token = req_to_token
|
||||
self.use_deep_gemm_prefill = use_deep_gemm_prefill
|
||||
self.use_deep_gemm_decode = use_deep_gemm_decode
|
||||
self.use_deep_select_decode = (
|
||||
not use_deep_gemm_decode and is_deep_select_supported()
|
||||
)
|
||||
|
||||
def topk_prefill(self, inputs: PrefillInputs) -> None:
|
||||
if self.use_deep_gemm_prefill:
|
||||
@@ -134,6 +141,18 @@ class FullTopKIndexer:
|
||||
)
|
||||
if d is None:
|
||||
return
|
||||
if self.use_deep_select_decode:
|
||||
metadata = inputs.paged_metadata
|
||||
topk_page_transform(
|
||||
d.scores,
|
||||
inputs.indexer.index_topk,
|
||||
page_table=metadata.page_table[: d.bs],
|
||||
page_size=metadata.compressed_page_size,
|
||||
end=d.lens.to(torch.int32),
|
||||
sorted_index=False,
|
||||
output_idx=inputs.out_page_indices[: d.bs],
|
||||
)
|
||||
return
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
|
||||
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
|
||||
@@ -148,6 +148,7 @@ def _random_page_table(rows: int, width: int, page_size: int):
|
||||
@pytest.mark.parametrize(
|
||||
"k,rows,width,with_end",
|
||||
[
|
||||
(512, 4, 64, True),
|
||||
(2048, 150, 33000, True),
|
||||
(1024, 2, 600_000, False),
|
||||
],
|
||||
@@ -161,7 +162,7 @@ def test_page_transform_matches_reference(k, rows, width, with_end):
|
||||
end = torch.randint(
|
||||
1, width + 1, (rows,), dtype=torch.int32, device="cuda", generator=g
|
||||
)
|
||||
end[0] = k // 2
|
||||
end[0] = min(k // 2, width)
|
||||
table, inverse = _random_page_table(rows, width, page_size)
|
||||
|
||||
slots = topk_page_transform(
|
||||
|
||||
@@ -904,6 +904,61 @@ class TestChunkedTopKMatchesUnchunked(CustomTestCase):
|
||||
self.assertTrue(torch.equal(run(rows_per_chunk), expected))
|
||||
|
||||
|
||||
class TestDeepSelectFullTopKDecode(CustomTestCase):
|
||||
@staticmethod
|
||||
def _decode_case():
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer import DecodeInputs
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import DecodeScores
|
||||
|
||||
scores = torch.tensor(
|
||||
[[1.0, 4.0, 2.0, 3.0], [3.0, 2.0, -torch.inf, -torch.inf]],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
lens = torch.tensor([4, 2], dtype=torch.int64)
|
||||
slots = torch.tensor([[40, 10, 30, 20], [80, 70, 0, 0]], dtype=torch.int64)
|
||||
metadata = SimpleNamespace(
|
||||
page_table=torch.tensor([[10], [20]], dtype=torch.int32),
|
||||
compressed_page_size=4,
|
||||
)
|
||||
page_indices = torch.full((2, 8), -1, dtype=torch.int32)
|
||||
inputs = DecodeInputs(
|
||||
indexer=SimpleNamespace(index_topk=8),
|
||||
layer_id=0,
|
||||
compress_ratio=1,
|
||||
freqs_cis=None,
|
||||
x=None,
|
||||
q_lora=None,
|
||||
positions=None,
|
||||
req_rows=torch.arange(2),
|
||||
paged_metadata=metadata,
|
||||
is_verify=False,
|
||||
out_page_indices=page_indices,
|
||||
)
|
||||
data = DecodeScores(bs=2, lmax=4, lens=lens, slots=slots, scores=scores)
|
||||
return inputs, data
|
||||
|
||||
def test_decode_uses_select_decode_fallback(self):
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer import full_topk as mod
|
||||
|
||||
inputs, data = self._decode_case()
|
||||
inputs.indexer.index_topk = 2
|
||||
indexer = object.__new__(mod.FullTopKIndexer)
|
||||
indexer.token_to_kv_pool = None
|
||||
indexer.req_to_token = None
|
||||
indexer.use_deep_gemm_decode = False
|
||||
indexer.use_deep_select_decode = False
|
||||
|
||||
with (
|
||||
patch.object(mod, "decode_scores", return_value=data),
|
||||
patch.object(mod, "topk_page_transform") as page_transform,
|
||||
patch.object(mod, "select_decode") as select_decode,
|
||||
):
|
||||
indexer.topk_decode(inputs)
|
||||
|
||||
page_transform.assert_not_called()
|
||||
select_decode.assert_called_once_with(inputs, data, 2)
|
||||
|
||||
|
||||
class TestChunkedPagedDecode(CustomTestCase):
|
||||
"""An eager decode whose paged-MQA metadata is split into row chunks (#40637):
|
||||
every DeepGEMM call must get its own chunk's schedule and every top-k its own
|
||||
|
||||
Reference in New Issue
Block a user