From b50fdeb0ef9be35916b88efb5a5df2cb8e5ac31d Mon Sep 17 00:00:00 2001 From: yuyu5333 <77156718+yuyu5333@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:46:12 +0800 Subject: [PATCH] [DeepSeek V4.1] Use DeepSelect for Hopper full top-k decode. (#41561) Co-authored-by: Claude Code --- .../attention/dsv4/v41_indexer/full_topk.py | 19 +++++++ .../kernels/ops/attention/test_deep_select.py | 3 +- .../unit/layers/test_dsv4_nonpaged_indexer.py | 55 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py index a2f317a9d3..fbf8e3acdd 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py @@ -7,6 +7,10 @@ from typing import TYPE_CHECKING import torch +from sglang.kernels.ops.attention.deep_select import ( + is_deep_select_supported, + topk_page_transform, +) from sglang.kernels.ops.attention.dsv4.index_logits import ( deep_gemm_fp4_paged_mqa_logits, ) @@ -47,6 +51,9 @@ class FullTopKIndexer: self.req_to_token = req_to_token self.use_deep_gemm_prefill = use_deep_gemm_prefill self.use_deep_gemm_decode = use_deep_gemm_decode + self.use_deep_select_decode = ( + not use_deep_gemm_decode and is_deep_select_supported() + ) def topk_prefill(self, inputs: PrefillInputs) -> None: if self.use_deep_gemm_prefill: @@ -134,6 +141,18 @@ class FullTopKIndexer: ) if d is None: return + if self.use_deep_select_decode: + metadata = inputs.paged_metadata + topk_page_transform( + d.scores, + inputs.indexer.index_topk, + page_table=metadata.page_table[: d.bs], + page_size=metadata.compressed_page_size, + end=d.lens.to(torch.int32), + sorted_index=False, + output_idx=inputs.out_page_indices[: d.bs], + ) + return select_decode(inputs, d, inputs.indexer.index_topk) def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None: diff --git a/test/registered/kernels/ops/attention/test_deep_select.py b/test/registered/kernels/ops/attention/test_deep_select.py index 607da66713..f51a6369e4 100644 --- a/test/registered/kernels/ops/attention/test_deep_select.py +++ b/test/registered/kernels/ops/attention/test_deep_select.py @@ -148,6 +148,7 @@ def _random_page_table(rows: int, width: int, page_size: int): @pytest.mark.parametrize( "k,rows,width,with_end", [ + (512, 4, 64, True), (2048, 150, 33000, True), (1024, 2, 600_000, False), ], @@ -161,7 +162,7 @@ def test_page_transform_matches_reference(k, rows, width, with_end): end = torch.randint( 1, width + 1, (rows,), dtype=torch.int32, device="cuda", generator=g ) - end[0] = k // 2 + end[0] = min(k // 2, width) table, inverse = _random_page_table(rows, width, page_size) slots = topk_page_transform( diff --git a/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py b/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py index 59e2d1b532..f8f21b4938 100644 --- a/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py +++ b/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py @@ -904,6 +904,61 @@ class TestChunkedTopKMatchesUnchunked(CustomTestCase): self.assertTrue(torch.equal(run(rows_per_chunk), expected)) +class TestDeepSelectFullTopKDecode(CustomTestCase): + @staticmethod + def _decode_case(): + from sglang.srt.layers.attention.dsv4.v41_indexer import DecodeInputs + from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import DecodeScores + + scores = torch.tensor( + [[1.0, 4.0, 2.0, 3.0], [3.0, 2.0, -torch.inf, -torch.inf]], + dtype=torch.float32, + ) + lens = torch.tensor([4, 2], dtype=torch.int64) + slots = torch.tensor([[40, 10, 30, 20], [80, 70, 0, 0]], dtype=torch.int64) + metadata = SimpleNamespace( + page_table=torch.tensor([[10], [20]], dtype=torch.int32), + compressed_page_size=4, + ) + page_indices = torch.full((2, 8), -1, dtype=torch.int32) + inputs = DecodeInputs( + indexer=SimpleNamespace(index_topk=8), + layer_id=0, + compress_ratio=1, + freqs_cis=None, + x=None, + q_lora=None, + positions=None, + req_rows=torch.arange(2), + paged_metadata=metadata, + is_verify=False, + out_page_indices=page_indices, + ) + data = DecodeScores(bs=2, lmax=4, lens=lens, slots=slots, scores=scores) + return inputs, data + + def test_decode_uses_select_decode_fallback(self): + from sglang.srt.layers.attention.dsv4.v41_indexer import full_topk as mod + + inputs, data = self._decode_case() + inputs.indexer.index_topk = 2 + indexer = object.__new__(mod.FullTopKIndexer) + indexer.token_to_kv_pool = None + indexer.req_to_token = None + indexer.use_deep_gemm_decode = False + indexer.use_deep_select_decode = False + + with ( + patch.object(mod, "decode_scores", return_value=data), + patch.object(mod, "topk_page_transform") as page_transform, + patch.object(mod, "select_decode") as select_decode, + ): + indexer.topk_decode(inputs) + + page_transform.assert_not_called() + select_decode.assert_called_once_with(inputs, data, 2) + + class TestChunkedPagedDecode(CustomTestCase): """An eager decode whose paged-MQA metadata is split into row chunks (#40637): every DeepGEMM call must get its own chunk's schedule and every top-k its own