fix(dsv4 fp4_indexer): 128B-align SM90 paged score rows for deep_select
fp4_index_logits_paged padded rows to 8 floats (32B) and fp4_index_logits_candidates did not pad at all, but the tvm-ffi deep_select topk kernel enforces kInputAlignmentBytes=128 on the input row stride. bs>1 decode/CUDA-graph capture died with 'Tensor stride for dimension 0 is not aligned to 128 bytes'. Candidates kernel now takes a separate OUT_STRIDE constexpr.
This commit is contained in:
@@ -948,9 +948,11 @@ def fp4_index_logits_paged(
|
||||
weights = weights.to(torch.bfloat16).contiguous()
|
||||
req = req.contiguous()
|
||||
lens = lens.contiguous()
|
||||
# Existing candidate-amax needs rows aligned to eight floats.
|
||||
# Rows are padded to 32 floats (128 bytes): the deep_select top-k kernel
|
||||
# requires 128-byte-aligned input row strides (kInputAlignmentBytes in
|
||||
# entry.cuh); the old eight-float alignment (candidate-amax) is subsumed.
|
||||
out = torch.empty(
|
||||
(batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32
|
||||
(batch, triton.cdiv(max_len, 32) * 32), device=q.device, dtype=torch.float32
|
||||
)
|
||||
if not batch or not max_len:
|
||||
return out[:, :max_len]
|
||||
@@ -995,6 +997,7 @@ def _fp4_index_logits_candidates_kernel(
|
||||
Blocks,
|
||||
Out,
|
||||
WIDTH: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
MAX_LEN: tl.constexpr,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
@@ -1038,7 +1041,7 @@ def _fp4_index_logits_candidates_kernel(
|
||||
H,
|
||||
64,
|
||||
)
|
||||
tl.store(Out + row * WIDTH + compact, score, mask=compact < WIDTH)
|
||||
tl.store(Out + row * OUT_STRIDE + compact, score, mask=compact < WIDTH)
|
||||
|
||||
|
||||
def fp4_index_logits_candidates(
|
||||
@@ -1071,9 +1074,12 @@ def fp4_index_logits_candidates(
|
||||
weights = weights.to(torch.bfloat16).contiguous()
|
||||
req, lens, blocks = req.contiguous(), lens.contiguous(), blocks.contiguous()
|
||||
width = blocks.shape[1] * 8
|
||||
out = torch.empty((batch, width), device=q.device, dtype=torch.float32)
|
||||
# Pad rows to 32 floats (128 bytes): the deep_select top-k kernel requires
|
||||
# 128-byte-aligned input row strides (kInputAlignmentBytes in entry.cuh).
|
||||
out_stride = triton.cdiv(width, 32) * 32
|
||||
out = torch.empty((batch, out_stride), device=q.device, dtype=torch.float32)
|
||||
if not batch or not width:
|
||||
return out
|
||||
return out[:, :width]
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
workers = min(triton.cdiv(width, 64), triton.cdiv(num_sms * 4, batch))
|
||||
_fp4_index_logits_candidates_kernel[(batch, workers)](
|
||||
@@ -1086,6 +1092,7 @@ def fp4_index_logits_candidates(
|
||||
blocks,
|
||||
out,
|
||||
width,
|
||||
out_stride,
|
||||
max_len,
|
||||
page_size,
|
||||
table.stride(0),
|
||||
@@ -1099,4 +1106,4 @@ def fp4_index_logits_candidates(
|
||||
64,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
return out[:, :width]
|
||||
|
||||
Reference in New Issue
Block a user