From fc9bdc8a3e60a0896e58d84ac261cc470ab4a24c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 29 Sep 2026 17:43:17 -0700 Subject: [PATCH] [MoE] Keep the moe_align_block_size pad fill inside sorted_token_ids (#41759) Co-authored-by: metamergebot Co-authored-by: oulgen --- .../benchmark/bench_moe_align_block_size.py | 2 + .../kernels/aot/csrc/moe/moe_align_kernel.cu | 6 +- .../kernels/aot/tests/test_moe_align.py | 2 + .../kernels/jit/csrc/moe/moe_align_kernel.cu | 12 +++- .../moe_lora_merged_align_kernel.cu | 12 +++- .../triton_utils/moe_align_block_size.py | 2 + .../ops/moe/test_moe_align_block_size.py | 58 +++++++++++++++++++ 7 files changed, 89 insertions(+), 5 deletions(-) diff --git a/python/sglang/kernels/aot/benchmark/bench_moe_align_block_size.py b/python/sglang/kernels/aot/benchmark/bench_moe_align_block_size.py index 3debde2f4d..349aae0b58 100644 --- a/python/sglang/kernels/aot/benchmark/bench_moe_align_block_size.py +++ b/python/sglang/kernels/aot/benchmark/bench_moe_align_block_size.py @@ -164,6 +164,8 @@ def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8): max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1) if topk_ids.numel() < num_experts + 1: max_num_tokens_padded_sgl = topk_ids.numel() * block_size + # The kernel pads sorted_ids with int4 stores. + max_num_tokens_padded_sgl = (max_num_tokens_padded_sgl + 3) // 4 * 4 sorted_ids_cuda = torch.empty( (max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device ) diff --git a/python/sglang/kernels/aot/csrc/moe/moe_align_kernel.cu b/python/sglang/kernels/aot/csrc/moe/moe_align_kernel.cu index f5ef1b4d07..67b67c9355 100644 --- a/python/sglang/kernels/aot/csrc/moe/moe_align_kernel.cu +++ b/python/sglang/kernels/aot/csrc/moe/moe_align_kernel.cu @@ -73,11 +73,15 @@ __global__ void moe_align_block_size_kernel( if (pad_sorted_token_ids) { Vec fill_vec; fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; - int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + // Whole vectors only; a partial last vector would write past the buffer. + int32_t total_vecs = max_num_tokens_padded / VEC_SIZE; Vec* out_ptr = reinterpret_cast(sorted_token_ids); for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { out_ptr[i] = fill_vec; } + for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) { + sorted_token_ids[i] = numel; + } } return; } diff --git a/python/sglang/kernels/aot/tests/test_moe_align.py b/python/sglang/kernels/aot/tests/test_moe_align.py index ea9861e6d5..f0cfe5e0a4 100644 --- a/python/sglang/kernels/aot/tests/test_moe_align.py +++ b/python/sglang/kernels/aot/tests/test_moe_align.py @@ -168,6 +168,8 @@ def test_moe_align_block_size_compare_implementations( max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) if topk_ids.numel() < num_experts + 1: max_num_tokens_padded = topk_ids.numel() * block_size + # The kernel pads sorted_ids with int4 stores. + max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4 sorted_ids_cuda = torch.empty( (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device diff --git a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu index 5c1726d4f0..85be81df64 100644 --- a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu +++ b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu @@ -71,11 +71,15 @@ __global__ void moe_align_block_size_kernel( if (pad_sorted_token_ids) { Vec fill_vec; fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; - int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + // Whole vectors only; a partial last vector would write past the buffer. + int32_t total_vecs = max_num_tokens_padded / VEC_SIZE; Vec* out_ptr = reinterpret_cast(sorted_token_ids); for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { out_ptr[i] = fill_vec; } + for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) { + sorted_token_ids[i] = numel; + } } return; } @@ -342,11 +346,15 @@ __global__ void moe_align_block_size_kernel_v2( if (pad_sorted_token_ids) { Vec fill_vec; fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; - int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + // Whole vectors only; a partial last vector would write past the buffer. + int32_t total_vecs = max_num_tokens_padded / VEC_SIZE; Vec* out_ptr = reinterpret_cast(sorted_token_ids); for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { out_ptr[i] = fill_vec; } + for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) { + sorted_token_ids[i] = numel; + } } return; } diff --git a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu b/python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu index a158e3bcb2..0bcfa0db58 100644 --- a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu +++ b/python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu @@ -152,11 +152,15 @@ __global__ void moe_align_block_size_kernel( if (pad_sorted_token_ids) { Vec fill_vec; fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; - int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + // Whole vectors only; a partial last vector would write past the buffer. + int32_t total_vecs = max_num_tokens_padded / VEC_SIZE; Vec* out_ptr = reinterpret_cast(sorted_token_ids); for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { out_ptr[i] = fill_vec; } + for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) { + sorted_token_ids[i] = numel; + } } return; } @@ -326,11 +330,15 @@ __global__ void fused_align_scatter_kernel( { Vec fill_vec; fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; - int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + // Whole vectors only; a partial last vector would write past the buffer. + int32_t total_vecs = max_num_tokens_padded / VEC_SIZE; Vec* out_ptr = reinterpret_cast(sorted_token_ids); for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { out_ptr[i] = fill_vec; } + for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) { + sorted_token_ids[i] = numel; + } } if (tid < num_experts) shared_counts[tid] = 0; __syncthreads(); diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py index 9c611a4682..b36b37ea5e 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py @@ -88,6 +88,8 @@ def moe_align_block_size( max_num_tokens_padded = topk_ids.numel() * block_size else: max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + # The align kernel pads sorted_ids with int4 stores. + max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4 sorted_ids = torch.empty( (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device diff --git a/test/registered/kernels/ops/moe/test_moe_align_block_size.py b/test/registered/kernels/ops/moe/test_moe_align_block_size.py index 326905ec61..f3acc05789 100644 --- a/test/registered/kernels/ops/moe/test_moe_align_block_size.py +++ b/test/registered/kernels/ops/moe/test_moe_align_block_size.py @@ -179,6 +179,8 @@ def test_moe_align_block_size_compare_implementations( max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) if topk_ids.numel() < num_experts + 1: max_num_tokens_padded = topk_ids.numel() * block_size + # The kernel pads sorted_ids with int4 stores. + max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4 sorted_ids_cuda = torch.empty( (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device @@ -294,6 +296,8 @@ def test_moe_align_block_size_v2_large_num_experts( max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) if topk_ids.numel() < num_experts + 1: max_num_tokens_padded = topk_ids.numel() * block_size + # The kernel pads sorted_ids with int4 stores. + max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4 sorted_ids_cuda = torch.empty( (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device @@ -359,5 +363,59 @@ def test_moe_align_block_size_v2_large_num_experts( ) +@pytest.mark.parametrize("remainder", [1, 2, 3]) +@pytest.mark.parametrize( + "num_experts,num_tokens", + [ + (32, 8), # small-batch kernel: <= 64 buckets and numel < 1024 + (128, 640), # generic kernel: <= 1024 buckets + (1025, 640), # v2 kernel: > 1024 buckets + ], +) +def test_moe_align_block_size_pad_fill_stays_in_bounds( + num_experts, num_tokens, remainder +): + """The pad fill writes sorted_token_ids with int4 stores; a buffer whose + length is not a multiple of 4 must be filled to its end and no further.""" + block_size, topk = 64, 8 + topk_ids = torch.randint( + 0, num_experts, (num_tokens, topk), dtype=torch.int32, device="cuda" + ) + needed = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + length = needed + (remainder - needed) % 4 + assert length % 4 == remainder + + # sorted_ids is a prefix view of a longer buffer; a fill that runs past the + # view lands in the guard words behind it. + backing = torch.full((length + 4,), -1, dtype=torch.int32, device="cuda") + sorted_ids = backing[:length] + expert_ids = torch.zeros( + (ceil_div(length, block_size),), dtype=torch.int32, device="cuda" + ) + num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda") + cumsum_buffer = torch.empty(num_experts + 2, dtype=torch.int32, device="cuda") + + moe_align_block_size( + topk_ids, + num_experts + 1, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + cumsum_buffer, + True, + ) + torch.cuda.synchronize() + + guard = backing[length:] + assert torch.equal(guard, torch.full_like(guard, -1)), ( + f"pad fill wrote past sorted_token_ids for length={length}: {guard.tolist()}" + ) + tail = sorted_ids[num_tokens_post_pad.item() :] + assert torch.equal(tail, torch.full_like(tail, topk_ids.numel())), ( + f"pad fill left part of the tail unfilled for length={length}" + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__]))