[MoE] Keep the moe_align_block_size pad fill inside sorted_token_ids (#41759)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: oulgen <oulgen@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-09-29 17:43:17 -07:00
committed by GitHub
co-authored by metamergebot oulgen
parent e5cec303ae
commit fc9bdc8a3e
7 changed files with 89 additions and 5 deletions
@@ -164,6 +164,8 @@ def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8):
max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded_sgl = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded_sgl = (max_num_tokens_padded_sgl + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device
)
@@ -73,11 +73,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -168,6 +168,8 @@ def test_moe_align_block_size_compare_implementations(
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
@@ -71,11 +71,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -342,11 +346,15 @@ __global__ void moe_align_block_size_kernel_v2(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -152,11 +152,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -326,11 +330,15 @@ __global__ void fused_align_scatter_kernel(
{
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
if (tid < num_experts) shared_counts[tid] = 0;
__syncthreads();
@@ -88,6 +88,8 @@ def moe_align_block_size(
max_num_tokens_padded = topk_ids.numel() * block_size
else:
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
# The align kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
@@ -179,6 +179,8 @@ def test_moe_align_block_size_compare_implementations(
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
@@ -294,6 +296,8 @@ def test_moe_align_block_size_v2_large_num_experts(
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
@@ -359,5 +363,59 @@ def test_moe_align_block_size_v2_large_num_experts(
)
@pytest.mark.parametrize("remainder", [1, 2, 3])
@pytest.mark.parametrize(
"num_experts,num_tokens",
[
(32, 8), # small-batch kernel: <= 64 buckets and numel < 1024
(128, 640), # generic kernel: <= 1024 buckets
(1025, 640), # v2 kernel: > 1024 buckets
],
)
def test_moe_align_block_size_pad_fill_stays_in_bounds(
num_experts, num_tokens, remainder
):
"""The pad fill writes sorted_token_ids with int4 stores; a buffer whose
length is not a multiple of 4 must be filled to its end and no further."""
block_size, topk = 64, 8
topk_ids = torch.randint(
0, num_experts, (num_tokens, topk), dtype=torch.int32, device="cuda"
)
needed = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
length = needed + (remainder - needed) % 4
assert length % 4 == remainder
# sorted_ids is a prefix view of a longer buffer; a fill that runs past the
# view lands in the guard words behind it.
backing = torch.full((length + 4,), -1, dtype=torch.int32, device="cuda")
sorted_ids = backing[:length]
expert_ids = torch.zeros(
(ceil_div(length, block_size),), dtype=torch.int32, device="cuda"
)
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda")
cumsum_buffer = torch.empty(num_experts + 2, dtype=torch.int32, device="cuda")
moe_align_block_size(
topk_ids,
num_experts + 1,
block_size,
sorted_ids,
expert_ids,
num_tokens_post_pad,
cumsum_buffer,
True,
)
torch.cuda.synchronize()
guard = backing[length:]
assert torch.equal(guard, torch.full_like(guard, -1)), (
f"pad fill wrote past sorted_token_ids for length={length}: {guard.tolist()}"
)
tail = sorted_ids[num_tokens_post_pad.item() :]
assert torch.equal(tail, torch.full_like(tail, topk_ids.numel())), (
f"pad fill left part of the tail unfilled for length={length}"
)
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))