[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)
Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
co-authored by
Aditya Suresh
parent
533020b500
commit
299a75ac5d
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
|
||||
|
||||
m.def("ggml_moe_get_block_size(int type) -> int");
|
||||
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
|
||||
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
|
||||
|
||||
/*
|
||||
* From csrc/mamba
|
||||
|
||||
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
|
||||
const int nrows_y,
|
||||
const int nrows_dst,
|
||||
const int top_k) {
|
||||
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
|
||||
|
||||
const int blocks_per_row_x = ncols_x / qk;
|
||||
const int blocks_per_col_y = nrows_y / QK8_1;
|
||||
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
|
||||
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
|
||||
}
|
||||
|
||||
const int exp_idx = expert_ids[blockIdx.y];
|
||||
if (exp_idx > 255 || exp_idx < 0) return;
|
||||
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
|
||||
if (exp_idx < 0) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
const int col_dst = token_offs[j / nwarps];
|
||||
if (col_dst >= ncols_dst) continue;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
const auto row_dst = row_dst_0 + threadIdx.x + i;
|
||||
if (row_dst < nrows_dst) {
|
||||
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (exp_idx > 255) return;
|
||||
|
||||
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
|
||||
const block_q8_1* y = (const block_q8_1*)(vy);
|
||||
|
||||
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
|
||||
if (row >= nrows) {
|
||||
return;
|
||||
}
|
||||
if (expert < 0) {
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
||||
|
||||
@@ -383,14 +383,17 @@ __global__ void Marlin(
|
||||
|
||||
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
int parallel = num_tokens_past_padded / moe_block_size;
|
||||
int num_valid_blocks = parallel;
|
||||
if constexpr (kIsEP) {
|
||||
for (int i = 0; i < parallel; i++) {
|
||||
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
|
||||
}
|
||||
// Every warp counts the same inactive blocks without a serial per-thread scan.
|
||||
int num_invalid_blocks = 0;
|
||||
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
|
||||
num_invalid_blocks += expert_ids_ptr[i] == -1;
|
||||
}
|
||||
int num_invalid_blocks = parallel - num_valid_blocks;
|
||||
parallel = num_valid_blocks;
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
|
||||
}
|
||||
parallel -= num_invalid_blocks;
|
||||
if (parallel == 0) return;
|
||||
|
||||
int k_tiles = prob_k / 16 / thread_k_blocks;
|
||||
int n_tiles = prob_n / 16 / thread_n_blocks;
|
||||
@@ -518,23 +521,18 @@ __global__ void Marlin(
|
||||
if (par_id >= parallel) return;
|
||||
|
||||
old_expert_id = expert_id;
|
||||
if constexpr (kIsEP) {
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
skip_count--;
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
expert_id = expert_ids_ptr[block_id];
|
||||
skip_count--;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
|
||||
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
|
||||
bool is_ep) {
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
int32_t neg_count = 0;
|
||||
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
// Padding and nonlocal routes are both inactive, independently of EP.
|
||||
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
|
||||
for (int e = tid; e <= num_experts; e += stride) {
|
||||
int32_t offset;
|
||||
|
||||
@@ -60,6 +60,16 @@ register_kernel(
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.shuffle_weighted_sum",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def moe_align_block_size(
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
|
||||
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
|
||||
use_int4_w4a16: tl.constexpr,
|
||||
use_int8_w8a16: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
|
||||
token_mask = offs_token < num_valid_tokens
|
||||
|
||||
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
|
||||
if filter_expert and off_experts == -1:
|
||||
# -----------------------------------------------------------
|
||||
# Write back zeros to the output when the expert is not
|
||||
# in the current expert parallel rank.
|
||||
if off_experts == -1:
|
||||
# Padding and nonlocal routes both use -1, including without EP.
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -377,12 +374,10 @@ def fused_moe_kernel(
|
||||
per_channel_quant: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
c_sorted: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
swap_ab: tl.constexpr,
|
||||
FUSE_ADD_TO_OUTPUT: tl.constexpr,
|
||||
MASK_OUTPUT: tl.constexpr,
|
||||
FUSE_SUM_ALL_REDUCE: tl.constexpr,
|
||||
LORA_PRESERVE_BASE: tl.constexpr,
|
||||
ROUTER_TOPK: tl.constexpr,
|
||||
FUSE_SWIGLU: tl.constexpr = False,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
@@ -450,16 +445,10 @@ def fused_moe_kernel(
|
||||
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
|
||||
off_experts = off_experts_i32.to(tl.int64)
|
||||
|
||||
if filter_expert and off_experts == -1:
|
||||
if FUSE_SWIGLU:
|
||||
# C is the half-width post-activation buffer here. Rows owned by a
|
||||
# filtered expert are never read (the down-GEMM CTA for this block
|
||||
# early-exits before loading A), and an N-wide zero store would run
|
||||
# past the row end into a neighboring token's data.
|
||||
return
|
||||
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
|
||||
# Write zeros only when this kernel owns the full output; the experimental LoRA
|
||||
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
|
||||
if off_experts == -1:
|
||||
# The down GEMM never reads inactive sorted or half-width intermediates.
|
||||
# Add/reduce outputs already hold their initial value and must stay intact.
|
||||
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
|
||||
lora_preserve_base: bool = False,
|
||||
fuse_swiglu: bool = False,
|
||||
) -> None:
|
||||
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
|
||||
assert topk_weights.stride(1) == 1
|
||||
assert sorted_token_ids.stride(0) == 1
|
||||
|
||||
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
|
||||
use_int4_w4a16=use_int4_w4a16,
|
||||
use_int8_w8a16=use_int8_w8a16,
|
||||
even_Ks=even_Ks,
|
||||
filter_expert=filter_expert,
|
||||
**config,
|
||||
)
|
||||
|
||||
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
|
||||
per_channel_quant=per_channel_quant,
|
||||
even_Ks=even_Ks,
|
||||
c_sorted=c_sorted,
|
||||
filter_expert=filter_expert,
|
||||
swap_ab=swap_ab,
|
||||
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
|
||||
MASK_OUTPUT=mask_output,
|
||||
LORA_PRESERVE_BASE=lora_preserve_base,
|
||||
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
|
||||
ROUTER_TOPK=router_topk,
|
||||
FUSE_SWIGLU=fuse_swiglu,
|
||||
|
||||
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
|
||||
num_tokens,
|
||||
stride_m,
|
||||
has_expert_map: tl.constexpr,
|
||||
is_ep: tl.constexpr,
|
||||
has_topk_ids: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
size: tl.constexpr,
|
||||
routed_scaling_factor: tl.constexpr,
|
||||
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
|
||||
if routed_scaling_factor != 1.0:
|
||||
b_val = b_val * routed_scaling_factor
|
||||
if has_expert_map:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
|
||||
mapped_id = tl.load(
|
||||
expert_map_ptr + tl.maximum(id_val, 0),
|
||||
mask=m_mask & (id_val >= 0),
|
||||
other=-1,
|
||||
)
|
||||
expert_mask = mapped_id >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
elif is_ep:
|
||||
elif has_topk_ids:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = id_val >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
|
||||
Shape: (num_tokens, top_k).
|
||||
outputs: Optional pre-allocated output tensor.
|
||||
Shape: (num_tokens, hidden_size).
|
||||
topk_ids: Optional indices of the top-k experts. Used when
|
||||
`expert_map` is provided. Shape: (num_tokens, top_k).
|
||||
topk_ids: Optional indices of the top-k experts. Negative ids always
|
||||
skip the corresponding input load. Shape: (num_tokens, top_k).
|
||||
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
||||
indicates an invalid token/expert pair that will be skipped.
|
||||
|
||||
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
|
||||
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
|
||||
# Keep is_ep as a compatible host argument; route validity follows ids.
|
||||
num_tokens, top_k, size = inputs.shape
|
||||
output_shape = (num_tokens, size)
|
||||
if outputs is None:
|
||||
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
|
||||
num_tokens,
|
||||
top_k * size,
|
||||
expert_map is not None,
|
||||
is_ep,
|
||||
topk_ids is not None,
|
||||
top_k,
|
||||
size,
|
||||
1.0 if routed_scaling_factor is None else routed_scaling_factor,
|
||||
|
||||
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Project routes, preserving inactive rows in a caller-provided output.
|
||||
|
||||
Inactive rows are zero when this function allocates the output.
|
||||
"""
|
||||
device = a.device
|
||||
|
||||
# Allocate output if not provided
|
||||
if c_or_none is not None:
|
||||
c = c_or_none
|
||||
else:
|
||||
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
|
||||
# Early return for zero-size M
|
||||
if size_m == 0:
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Reduce routed rows while skipping inactive permutation entries."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
|
||||
def _shuffle_weighted_sum_kernel(
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
width: tl.constexpr,
|
||||
rows,
|
||||
topk: tl.constexpr,
|
||||
input_stride: tl.constexpr,
|
||||
output_stride: tl.constexpr,
|
||||
weight_stride0,
|
||||
weight_stride1,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
total = tl.full((BLOCK,), 0, tl.float32)
|
||||
for slot in range(topk):
|
||||
row = tl.load(permutation + token * topk + slot).to(tl.int64)
|
||||
active = (row >= 0) & (row < rows)
|
||||
values = tl.load(
|
||||
inputs + row * input_stride + columns,
|
||||
mask=active & (columns < width),
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
factor = tl.load(
|
||||
weights + token * weight_stride0 + slot * weight_stride1,
|
||||
mask=active,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
total += values * factor
|
||||
tl.store(
|
||||
output + token.to(tl.int64) * output_stride + columns,
|
||||
total,
|
||||
mask=columns < width,
|
||||
)
|
||||
|
||||
|
||||
def shuffle_weighted_sum(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
permutation: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
) -> None:
|
||||
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
|
||||
assert inputs.ndim == output.ndim == weights.ndim == 2
|
||||
assert inputs.shape[1] == output.shape[1]
|
||||
assert inputs.stride(1) == output.stride(1) == 1
|
||||
assert weights.shape[0] == output.shape[0]
|
||||
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
|
||||
if output.numel() == 0:
|
||||
return
|
||||
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
output.shape[1],
|
||||
inputs.shape[0],
|
||||
weights.shape[1],
|
||||
inputs.stride(0),
|
||||
output.stride(0),
|
||||
weights.stride(0),
|
||||
weights.stride(1),
|
||||
BLOCK=256,
|
||||
)
|
||||
Reference in New Issue
Block a user