fix(dsv4 fp4_indexer): 128B-align SM90 paged score rows for deep_select

fp4_index_logits_paged padded rows to 8 floats (32B) and
fp4_index_logits_candidates did not pad at all, but the tvm-ffi
deep_select topk kernel enforces kInputAlignmentBytes=128 on the input
row stride. bs>1 decode/CUDA-graph capture died with
'Tensor stride for dimension 0 is not aligned to 128 bytes'.
Candidates kernel now takes a separate OUT_STRIDE constexpr.
This commit is contained in:
2026-10-19 20:55:01 +08:00
parent 2aae0cb4ee
commit f9056f0fcd
@@ -948,9 +948,11 @@ def fp4_index_logits_paged(
weights = weights.to(torch.bfloat16).contiguous()
req = req.contiguous()
lens = lens.contiguous()
# Existing candidate-amax needs rows aligned to eight floats.
# Rows are padded to 32 floats (128 bytes): the deep_select top-k kernel
# requires 128-byte-aligned input row strides (kInputAlignmentBytes in
# entry.cuh); the old eight-float alignment (candidate-amax) is subsumed.
out = torch.empty(
(batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32
(batch, triton.cdiv(max_len, 32) * 32), device=q.device, dtype=torch.float32
)
if not batch or not max_len:
return out[:, :max_len]
@@ -995,6 +997,7 @@ def _fp4_index_logits_candidates_kernel(
Blocks,
Out,
WIDTH: tl.constexpr,
OUT_STRIDE: tl.constexpr,
MAX_LEN: tl.constexpr,
PAGE_SIZE: tl.constexpr,
CACHE_STRIDE: tl.constexpr,
@@ -1038,7 +1041,7 @@ def _fp4_index_logits_candidates_kernel(
H,
64,
)
tl.store(Out + row * WIDTH + compact, score, mask=compact < WIDTH)
tl.store(Out + row * OUT_STRIDE + compact, score, mask=compact < WIDTH)
def fp4_index_logits_candidates(
@@ -1071,9 +1074,12 @@ def fp4_index_logits_candidates(
weights = weights.to(torch.bfloat16).contiguous()
req, lens, blocks = req.contiguous(), lens.contiguous(), blocks.contiguous()
width = blocks.shape[1] * 8
out = torch.empty((batch, width), device=q.device, dtype=torch.float32)
# Pad rows to 32 floats (128 bytes): the deep_select top-k kernel requires
# 128-byte-aligned input row strides (kInputAlignmentBytes in entry.cuh).
out_stride = triton.cdiv(width, 32) * 32
out = torch.empty((batch, out_stride), device=q.device, dtype=torch.float32)
if not batch or not width:
return out
return out[:, :width]
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
workers = min(triton.cdiv(width, 64), triton.cdiv(num_sms * 4, batch))
_fp4_index_logits_candidates_kernel[(batch, workers)](
@@ -1086,6 +1092,7 @@ def fp4_index_logits_candidates(
blocks,
out,
width,
out_stride,
max_len,
page_size,
table.stride(0),
@@ -1099,4 +1106,4 @@ def fp4_index_logits_candidates(
64,
num_warps=4,
)
return out
return out[:, :width]