[Fix] DeepSeek-V4.1: Align paged indexer score rows for DeepSelect decode (#44210)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-14 17:31:19 +08:00
committed by GitHub
co-authored by Mohammad Angkad
parent df1a089cd3
commit c82e8bd5fe
4 changed files with 30 additions and 16 deletions
@@ -5,6 +5,8 @@ import triton
import triton.language as tl
from triton.language.extra import libdevice
from sglang.kernels.jit.utils import aligned_new_empty
from sglang.kernels.ops.attention.deep_select import get_input_stride_alignment_bytes
from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR
from sglang.srt.runtime_context import get_platform
@@ -910,12 +912,15 @@ def fp4_index_logits_paged(
weights = weights.to(torch.bfloat16).contiguous()
req = req.contiguous()
lens = lens.contiguous()
# Existing candidate-amax needs rows aligned to eight floats.
out = torch.empty(
(batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32
# Rows aligned for DeepSelect top-k (128B) also suit candidate-amax (8 floats).
out = aligned_new_empty(
shape=(batch, max_len),
dtype=torch.float32,
device=q.device,
alignment_bytes=get_input_stride_alignment_bytes(),
)
if not batch or not max_len:
return out[:, :max_len]
return out
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
workers = min(triton.cdiv(max_len, 64), triton.cdiv(num_sms * 4, batch))
_fp4_index_logits_paged_kernel[(batch, workers)](
@@ -943,4 +948,4 @@ def fp4_index_logits_paged(
BLOCK_L=64,
num_warps=4,
)
return out[:, :max_len]
return out
@@ -138,6 +138,7 @@ class FullTopKIndexer:
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
compute_topk_plan=not self.use_deep_select_decode,
)
if d is None:
return
@@ -367,7 +367,7 @@ class PagedDecodeScores(msgspec.Struct, frozen=True):
req: torch.Tensor
req_to_token: torch.Tensor
ratio: int
plan: torch.Tensor
plan: Optional[torch.Tensor] # None when the caller skips the top-k v2 plan
has_candidate_mask: bool
@@ -377,6 +377,7 @@ def decode_scores(
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
candidate_mask: Optional[torch.Tensor] = None,
compute_topk_plan: bool = True,
) -> Optional[DecodeScores | PagedDecodeScores]:
"""None when there is no row, or nothing visible yet."""
pool = token_to_kv_pool
@@ -405,7 +406,7 @@ def decode_scores(
if paged:
lens = lens.to(torch.int32)
# Prepare the device-side plan before scoring, away from its top-k consumer.
plan = plan_topk_v2(lens)
plan = plan_topk_v2(lens) if compute_topk_plan else None
scores = fp4_index_logits_paged(
q,
weights,
@@ -445,6 +446,7 @@ def select_decode(
) -> None:
k = min(topk, d.lmax)
if isinstance(d, PagedDecodeScores):
assert d.plan is not None, "paged select_decode needs the top-k v2 plan"
idx = torch.empty((d.bs, k), dtype=torch.int32, device=d.scores.device)
topk_transform_paged_v2(d.scores, d.lens, None, idx, 1, d.plan)
finish_paged_indexer_topk(