diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index 596130c389..15ac5acb01 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -948,9 +948,11 @@ def fp4_index_logits_paged( weights = weights.to(torch.bfloat16).contiguous() req = req.contiguous() lens = lens.contiguous() - # Existing candidate-amax needs rows aligned to eight floats. + # Rows are padded to 32 floats (128 bytes): the deep_select top-k kernel + # requires 128-byte-aligned input row strides (kInputAlignmentBytes in + # entry.cuh); the old eight-float alignment (candidate-amax) is subsumed. out = torch.empty( - (batch, triton.cdiv(max_len, 8) * 8), device=q.device, dtype=torch.float32 + (batch, triton.cdiv(max_len, 32) * 32), device=q.device, dtype=torch.float32 ) if not batch or not max_len: return out[:, :max_len] @@ -995,6 +997,7 @@ def _fp4_index_logits_candidates_kernel( Blocks, Out, WIDTH: tl.constexpr, + OUT_STRIDE: tl.constexpr, MAX_LEN: tl.constexpr, PAGE_SIZE: tl.constexpr, CACHE_STRIDE: tl.constexpr, @@ -1038,7 +1041,7 @@ def _fp4_index_logits_candidates_kernel( H, 64, ) - tl.store(Out + row * WIDTH + compact, score, mask=compact < WIDTH) + tl.store(Out + row * OUT_STRIDE + compact, score, mask=compact < WIDTH) def fp4_index_logits_candidates( @@ -1071,9 +1074,12 @@ def fp4_index_logits_candidates( weights = weights.to(torch.bfloat16).contiguous() req, lens, blocks = req.contiguous(), lens.contiguous(), blocks.contiguous() width = blocks.shape[1] * 8 - out = torch.empty((batch, width), device=q.device, dtype=torch.float32) + # Pad rows to 32 floats (128 bytes): the deep_select top-k kernel requires + # 128-byte-aligned input row strides (kInputAlignmentBytes in entry.cuh). + out_stride = triton.cdiv(width, 32) * 32 + out = torch.empty((batch, out_stride), device=q.device, dtype=torch.float32) if not batch or not width: - return out + return out[:, :width] num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count workers = min(triton.cdiv(width, 64), triton.cdiv(num_sms * 4, batch)) _fp4_index_logits_candidates_kernel[(batch, workers)]( @@ -1086,6 +1092,7 @@ def fp4_index_logits_candidates( blocks, out, width, + out_stride, max_len, page_size, table.stride(0), @@ -1099,4 +1106,4 @@ def fp4_index_logits_candidates( 64, num_warps=4, ) - return out + return out[:, :width]