[MoE] Keep the moe_align_block_size pad fill inside sorted_token_ids (#41759)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: oulgen <oulgen@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
oulgen
parent
e5cec303ae
commit
fc9bdc8a3e
@@ -164,6 +164,8 @@ def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8):
|
||||
max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded_sgl = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded_sgl = (max_num_tokens_padded_sgl + 3) // 4 * 4
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
|
||||
@@ -73,11 +73,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -168,6 +168,8 @@ def test_moe_align_block_size_compare_implementations(
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
|
||||
@@ -71,11 +71,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -342,11 +346,15 @@ __global__ void moe_align_block_size_kernel_v2(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -152,11 +152,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -326,11 +330,15 @@ __global__ void fused_align_scatter_kernel(
|
||||
{
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
if (tid < num_experts) shared_counts[tid] = 0;
|
||||
__syncthreads();
|
||||
|
||||
@@ -88,6 +88,8 @@ def moe_align_block_size(
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
else:
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
# The align kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
|
||||
Reference in New Issue
Block a user