From f9056f0fcdabfbb35f69705afd6c831f4777cbfa Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 19 Oct 2026 20:55:01 +0800 Subject: [PATCH] fix(dsv4 fp4_indexer): 128B-align SM90 paged score rows for deep_select fp4_index_logits_paged padded rows to 8 floats (32B) and fp4_index_logits_candidates did not pad at all, but the tvm-ffi deep_select topk kernel enforces kInputAlignmentBytes=128 on the input row stride. bs>1 decode/CUDA-graph capture died with 'Tensor stride for dimension 0 is not aligned to 128 bytes'. Candidates kernel now takes a separate OUT_STRIDE constexpr. --- .../kernels/ops/attention/dsv4/fp4_indexer.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index 596130c389..15ac5acb01 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -948,9 +948,11 @@ def fp4_index_logits_paged( weights = weights.to(torch.bfloat16).contiguous() req = req.contiguous() lens = lens.contiguous() - # Existing candidate-amax needs rows aligned to eight floats. + # Rows are padded to 32 floats (128 bytes): the deep_select top-k kernel + # requires 128-byte-aligned input row strides (kInputAlignmentBytes in + # entry.cuh); the old eight-float alignment (candidate-amax) is subsumed. out = torch.empty( - (batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32 + (batch, triton.cdiv(max_len, 32) * 32), device=q.device, dtype=torch.float32 ) if not batch or not max_len: return out[:, :max_len] @@ -995,6 +997,7 @@ def _fp4_index_logits_candidates_kernel( Blocks, Out, WIDTH: tl.constexpr, + OUT_STRIDE: tl.constexpr, MAX_LEN: tl.constexpr, PAGE_SIZE: tl.constexpr, CACHE_STRIDE: tl.constexpr, @@ -1038,7 +1041,7 @@ def _fp4_index_logits_candidates_kernel( H, 64, ) - tl.store(Out + row * WIDTH + compact, score, mask=compact < WIDTH) + tl.store(Out + row * OUT_STRIDE + compact, score, mask=compact < WIDTH) def fp4_index_logits_candidates( @@ -1071,9 +1074,12 @@ def fp4_index_logits_candidates( weights = weights.to(torch.bfloat16).contiguous() req, lens, blocks = req.contiguous(), lens.contiguous(), blocks.contiguous() width = blocks.shape[1] * 8 - out = torch.empty((batch, width), device=q.device, dtype=torch.float32) + # Pad rows to 32 floats (128 bytes): the deep_select top-k kernel requires + # 128-byte-aligned input row strides (kInputAlignmentBytes in entry.cuh). + out_stride = triton.cdiv(width, 32) * 32 + out = torch.empty((batch, out_stride), device=q.device, dtype=torch.float32) if not batch or not width: - return out + return out[:, :width] num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count workers = min(triton.cdiv(width, 64), triton.cdiv(num_sms * 4, batch)) _fp4_index_logits_candidates_kernel[(batch, workers)]( @@ -1086,6 +1092,7 @@ def fp4_index_logits_candidates( blocks, out, width, + out_stride, max_len, page_size, table.stride(0), @@ -1099,4 +1106,4 @@ def fp4_index_logits_candidates( 64, num_warps=4, ) - return out + return out[:, :width]