From c82e8bd5fe9cad787c2cd3ccf067a1ba5b95e0c4 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:31:19 +0800 Subject: [PATCH] [Fix] DeepSeek-V4.1: Align paged indexer score rows for DeepSelect decode (#44210) Co-authored-by: Mohammad Angkad --- .../kernels/ops/attention/dsv4/fp4_indexer.py | 15 ++++++++---- .../attention/dsv4/v41_indexer/full_topk.py | 1 + .../attention/dsv4/v41_indexer/scoring.py | 6 +++-- .../kernels/ops/attention/test_fp4_indexer.py | 24 ++++++++++++------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index 213d97c2b9..4bcde3084b 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -5,6 +5,8 @@ import triton import triton.language as tl from triton.language.extra import libdevice +from sglang.kernels.jit.utils import aligned_new_empty +from sglang.kernels.ops.attention.deep_select import get_input_stride_alignment_bytes from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR from sglang.srt.runtime_context import get_platform @@ -910,12 +912,15 @@ def fp4_index_logits_paged( weights = weights.to(torch.bfloat16).contiguous() req = req.contiguous() lens = lens.contiguous() - # Existing candidate-amax needs rows aligned to eight floats. - out = torch.empty( - (batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32 + # Rows aligned for DeepSelect top-k (128B) also suit candidate-amax (8 floats). + out = aligned_new_empty( + shape=(batch, max_len), + dtype=torch.float32, + device=q.device, + alignment_bytes=get_input_stride_alignment_bytes(), ) if not batch or not max_len: - return out[:, :max_len] + return out num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count workers = min(triton.cdiv(max_len, 64), triton.cdiv(num_sms * 4, batch)) _fp4_index_logits_paged_kernel[(batch, workers)]( @@ -943,4 +948,4 @@ def fp4_index_logits_paged( BLOCK_L=64, num_warps=4, ) - return out[:, :max_len] + return out diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py index fbf8e3acdd..883cf05757 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py @@ -138,6 +138,7 @@ class FullTopKIndexer: inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, req_to_token=self.req_to_token, + compute_topk_plan=not self.use_deep_select_decode, ) if d is None: return diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index ea90f9a5cd..487d1b96a7 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -367,7 +367,7 @@ class PagedDecodeScores(msgspec.Struct, frozen=True): req: torch.Tensor req_to_token: torch.Tensor ratio: int - plan: torch.Tensor + plan: Optional[torch.Tensor] # None when the caller skips the top-k v2 plan has_candidate_mask: bool @@ -377,6 +377,7 @@ def decode_scores( token_to_kv_pool: DeepSeekV4TokenToKVPool, req_to_token: torch.Tensor, candidate_mask: Optional[torch.Tensor] = None, + compute_topk_plan: bool = True, ) -> Optional[DecodeScores | PagedDecodeScores]: """None when there is no row, or nothing visible yet.""" pool = token_to_kv_pool @@ -405,7 +406,7 @@ def decode_scores( if paged: lens = lens.to(torch.int32) # Prepare the device-side plan before scoring, away from its top-k consumer. - plan = plan_topk_v2(lens) + plan = plan_topk_v2(lens) if compute_topk_plan else None scores = fp4_index_logits_paged( q, weights, @@ -445,6 +446,7 @@ def select_decode( ) -> None: k = min(topk, d.lmax) if isinstance(d, PagedDecodeScores): + assert d.plan is not None, "paged select_decode needs the top-k v2 plan" idx = torch.empty((d.bs, k), dtype=torch.int32, device=d.scores.device) topk_transform_paged_v2(d.scores, d.lens, None, idx, 1, d.plan) finish_paged_indexer_topk( diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer.py b/test/registered/kernels/ops/attention/test_fp4_indexer.py index dd9ab01175..42154a7d48 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer.py @@ -615,6 +615,14 @@ def test_hopper_indexer_backends_replay(ratio): req_table[:, : width * ratio : ratio] = (slots[::6] * ratio).int() positions = (lens * ratio - 1).clamp_min(0) pool = SimpleNamespace(get_index_k_with_scale_buffer=lambda layer: table) + # A unit page keeps the per-row table valid under the shuffled slots below. + page_table = torch.empty((batch, capacity), device=q.device, dtype=torch.int32) + page_table.copy_(req_table[req.long(), ::ratio] // ratio) + metadata = SimpleNamespace( + max_compressed_seq_len=capacity, + page_table=page_table, + compressed_page_size=1, + ) indexer = SimpleNamespace( queries=lambda q, freqs: q, head_weights=lambda x: x, index_topk=topk ) @@ -626,27 +634,24 @@ def test_hopper_indexer_backends_replay(ratio): x=weights, positions=positions, req_rows=req, - paged_metadata=SimpleNamespace(max_compressed_seq_len=capacity), + paged_metadata=metadata, is_verify=True, ) kwargs = dict( token_to_kv_pool=pool, req_to_token=req_table, use_deep_gemm_prefill=False ) full = FullTopKIndexer(**kwargs, use_deep_gemm_decode=False) - # This replay test checks the paged select_decode output contract; DeepSelect - # decode is covered by test_deep_select.py. - full.use_deep_select_decode = False candidates = DenseBlocksBackend( **kwargs, candidate_topk_blocks=4, candidate_block_size=8 ) + # DeepSelect fills a full top-k buffer of exactly live rows by topk, as at + # runtime; the candidate paths also clear padded rows and columns. outputs = [ SimpleNamespace( - page_indices=torch.empty( - (batch + 1, topk + 8), device=q.device, dtype=torch.int32 - ), + page_indices=torch.empty(shape, device=q.device, dtype=torch.int32), raw_indices=None, ) - for _ in range(3) + for shape in ((batch, topk), (batch + 1, topk + 8), (batch + 1, topk + 8)) ] decode_inputs = [ DecodeInputs( @@ -689,6 +694,7 @@ def test_hopper_indexer_backends_replay(ratio): positions.copy_(lens * ratio - 1) req.copy_(req.roll(1)) req_table[:, : width * ratio : ratio] = (slots[::6].roll(step, 1) * ratio).int() + page_table.copy_(req_table[req.long(), ::ratio] // ratio) q.neg_() for scores in captured_scores: scores.fill_(torch.inf) @@ -744,7 +750,7 @@ def test_hopper_indexer_backends_replay(ratio): expected_raw = torch.full_like(out.raw_indices[row], -1) expected_raw[:count] = selected.sort().values.int() torch.testing.assert_close(out.raw_indices[row], expected_raw) - for out in outputs: + for out in outputs[1:]: assert (out.page_indices[batch] == -1).all().item() if out.raw_indices is not None: assert (out.raw_indices[batch] == -1).all().item()