[MoE] Keep the moe_align_block_size pad fill inside sorted_token_ids (#41759)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: oulgen <oulgen@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
oulgen
parent
e5cec303ae
commit
fc9bdc8a3e
@@ -164,6 +164,8 @@ def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8):
|
||||
max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded_sgl = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded_sgl = (max_num_tokens_padded_sgl + 3) // 4 * 4
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
|
||||
@@ -73,11 +73,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -168,6 +168,8 @@ def test_moe_align_block_size_compare_implementations(
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
|
||||
@@ -71,11 +71,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -342,11 +346,15 @@ __global__ void moe_align_block_size_kernel_v2(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -152,11 +152,15 @@ __global__ void moe_align_block_size_kernel(
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -326,11 +330,15 @@ __global__ void fused_align_scatter_kernel(
|
||||
{
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
// Whole vectors only; a partial last vector would write past the buffer.
|
||||
int32_t total_vecs = max_num_tokens_padded / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
for (int32_t i = total_vecs * VEC_SIZE + threadIdx.x; i < max_num_tokens_padded; i += blockDim.x) {
|
||||
sorted_token_ids[i] = numel;
|
||||
}
|
||||
}
|
||||
if (tid < num_experts) shared_counts[tid] = 0;
|
||||
__syncthreads();
|
||||
|
||||
@@ -88,6 +88,8 @@ def moe_align_block_size(
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
else:
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
# The align kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
|
||||
@@ -179,6 +179,8 @@ def test_moe_align_block_size_compare_implementations(
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
@@ -294,6 +296,8 @@ def test_moe_align_block_size_v2_large_num_experts(
|
||||
max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
if topk_ids.numel() < num_experts + 1:
|
||||
max_num_tokens_padded = topk_ids.numel() * block_size
|
||||
# The kernel pads sorted_ids with int4 stores.
|
||||
max_num_tokens_padded = (max_num_tokens_padded + 3) // 4 * 4
|
||||
|
||||
sorted_ids_cuda = torch.empty(
|
||||
(max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device
|
||||
@@ -359,5 +363,59 @@ def test_moe_align_block_size_v2_large_num_experts(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remainder", [1, 2, 3])
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,num_tokens",
|
||||
[
|
||||
(32, 8), # small-batch kernel: <= 64 buckets and numel < 1024
|
||||
(128, 640), # generic kernel: <= 1024 buckets
|
||||
(1025, 640), # v2 kernel: > 1024 buckets
|
||||
],
|
||||
)
|
||||
def test_moe_align_block_size_pad_fill_stays_in_bounds(
|
||||
num_experts, num_tokens, remainder
|
||||
):
|
||||
"""The pad fill writes sorted_token_ids with int4 stores; a buffer whose
|
||||
length is not a multiple of 4 must be filled to its end and no further."""
|
||||
block_size, topk = 64, 8
|
||||
topk_ids = torch.randint(
|
||||
0, num_experts, (num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
needed = topk_ids.numel() + (num_experts + 1) * (block_size - 1)
|
||||
length = needed + (remainder - needed) % 4
|
||||
assert length % 4 == remainder
|
||||
|
||||
# sorted_ids is a prefix view of a longer buffer; a fill that runs past the
|
||||
# view lands in the guard words behind it.
|
||||
backing = torch.full((length + 4,), -1, dtype=torch.int32, device="cuda")
|
||||
sorted_ids = backing[:length]
|
||||
expert_ids = torch.zeros(
|
||||
(ceil_div(length, block_size),), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device="cuda")
|
||||
cumsum_buffer = torch.empty(num_experts + 2, dtype=torch.int32, device="cuda")
|
||||
|
||||
moe_align_block_size(
|
||||
topk_ids,
|
||||
num_experts + 1,
|
||||
block_size,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
cumsum_buffer,
|
||||
True,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
guard = backing[length:]
|
||||
assert torch.equal(guard, torch.full_like(guard, -1)), (
|
||||
f"pad fill wrote past sorted_token_ids for length={length}: {guard.tolist()}"
|
||||
)
|
||||
tail = sorted_ids[num_tokens_post_pad.item() :]
|
||||
assert torch.equal(tail, torch.full_like(tail, topk_ids.numel())), (
|
||||
f"pad fill left part of the tail unfilled for length={length}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
Reference in New Issue
Block a user