[MoE] Keep the moe_align_block_size pad fill inside sorted_token_ids (#41759)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: oulgen <oulgen@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-09-29 17:43:17 -07:00
committed by GitHub
co-authored by metamergebot oulgen
parent e5cec303ae
commit fc9bdc8a3e
7 changed files with 89 additions and 5 deletions
@@ -164,6 +164,8 @@ def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8):
max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded_sgl = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded_sgl = (max_num_tokens_padded_sgl + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device
)
@@ -73,11 +73,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -168,6 +168,8 @@ def test_moe_align_block_size_compare_implementations(
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
if topk_ids.numel() < num_experts + 1:
max_num_tokens_padded = topk_ids.numel() * block_size
# The kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids_cuda = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
@@ -71,11 +71,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -342,11 +346,15 @@ __global__ void moe_align_block_size_kernel_v2(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -152,11 +152,15 @@ __global__ void moe_align_block_size_kernel(
if (pad_sorted_token_ids) {
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
return;
}
@@ -326,11 +330,15 @@ __global__ void fused_align_scatter_kernel(
{
Vec fill_vec;
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
// Whole vectors only; a partial last vector would write past the buffer.
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
out_ptr[i] = fill_vec;
}
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
sorted_token_ids[i] = numel;
}
}
if (tid < num_experts) shared_counts[tid] = 0;
__syncthreads();
@@ -88,6 +88,8 @@ def moe_align_block_size(
max_num_tokens_padded = topk_ids.numel() * block_size
else:
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
# The align kernel pads sorted_ids with int4 stores.
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
sorted_ids = torch.empty(
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device