[Fix] DeepSeek-V4.1: Align paged indexer score rows for DeepSelect decode (#44210)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
df1a089cd3
commit
c82e8bd5fe
@@ -5,6 +5,8 @@ import triton
|
||||
import triton.language as tl
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.kernels.jit.utils import aligned_new_empty
|
||||
from sglang.kernels.ops.attention.deep_select import get_input_stride_alignment_bytes
|
||||
from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
@@ -910,12 +912,15 @@ def fp4_index_logits_paged(
|
||||
weights = weights.to(torch.bfloat16).contiguous()
|
||||
req = req.contiguous()
|
||||
lens = lens.contiguous()
|
||||
# Existing candidate-amax needs rows aligned to eight floats.
|
||||
out = torch.empty(
|
||||
(batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32
|
||||
# Rows aligned for DeepSelect top-k (128B) also suit candidate-amax (8 floats).
|
||||
out = aligned_new_empty(
|
||||
shape=(batch, max_len),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
alignment_bytes=get_input_stride_alignment_bytes(),
|
||||
)
|
||||
if not batch or not max_len:
|
||||
return out[:, :max_len]
|
||||
return out
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
workers = min(triton.cdiv(max_len, 64), triton.cdiv(num_sms * 4, batch))
|
||||
_fp4_index_logits_paged_kernel[(batch, workers)](
|
||||
@@ -943,4 +948,4 @@ def fp4_index_logits_paged(
|
||||
BLOCK_L=64,
|
||||
num_warps=4,
|
||||
)
|
||||
return out[:, :max_len]
|
||||
return out
|
||||
|
||||
@@ -138,6 +138,7 @@ class FullTopKIndexer:
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
compute_topk_plan=not self.use_deep_select_decode,
|
||||
)
|
||||
if d is None:
|
||||
return
|
||||
|
||||
@@ -367,7 +367,7 @@ class PagedDecodeScores(msgspec.Struct, frozen=True):
|
||||
req: torch.Tensor
|
||||
req_to_token: torch.Tensor
|
||||
ratio: int
|
||||
plan: torch.Tensor
|
||||
plan: Optional[torch.Tensor] # None when the caller skips the top-k v2 plan
|
||||
has_candidate_mask: bool
|
||||
|
||||
|
||||
@@ -377,6 +377,7 @@ def decode_scores(
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
candidate_mask: Optional[torch.Tensor] = None,
|
||||
compute_topk_plan: bool = True,
|
||||
) -> Optional[DecodeScores | PagedDecodeScores]:
|
||||
"""None when there is no row, or nothing visible yet."""
|
||||
pool = token_to_kv_pool
|
||||
@@ -405,7 +406,7 @@ def decode_scores(
|
||||
if paged:
|
||||
lens = lens.to(torch.int32)
|
||||
# Prepare the device-side plan before scoring, away from its top-k consumer.
|
||||
plan = plan_topk_v2(lens)
|
||||
plan = plan_topk_v2(lens) if compute_topk_plan else None
|
||||
scores = fp4_index_logits_paged(
|
||||
q,
|
||||
weights,
|
||||
@@ -445,6 +446,7 @@ def select_decode(
|
||||
) -> None:
|
||||
k = min(topk, d.lmax)
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
assert d.plan is not None, "paged select_decode needs the top-k v2 plan"
|
||||
idx = torch.empty((d.bs, k), dtype=torch.int32, device=d.scores.device)
|
||||
topk_transform_paged_v2(d.scores, d.lens, None, idx, 1, d.plan)
|
||||
finish_paged_indexer_topk(
|
||||
|
||||
Reference in New Issue
Block a user