[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)

Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
Cheng Wan
2026-10-20 16:26:15 -07:00
committed by GitHub
co-authored by Aditya Suresh
parent 533020b500
commit 299a75ac5d
83 changed files with 3915 additions and 641 deletions
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
m.def("ggml_moe_get_block_size(int type) -> int");
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
/*
* From csrc/mamba
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
const int nrows_y,
const int nrows_dst,
const int top_k) {
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
const int blocks_per_row_x = ncols_x / qk;
const int blocks_per_col_y = nrows_y / QK8_1;
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
}
const int exp_idx = expert_ids[blockIdx.y];
if (exp_idx > 255 || exp_idx < 0) return;
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
if (exp_idx < 0) {
#pragma unroll
for (int j = 0; j < mmq_x; j += nwarps) {
const int col_dst = token_offs[j / nwarps];
if (col_dst >= ncols_dst) continue;
#pragma unroll
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
const auto row_dst = row_dst_0 + threadIdx.x + i;
if (row_dst < nrows_dst) {
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
}
}
}
return;
}
if (exp_idx > 255) return;
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
const block_q8_1* y = (const block_q8_1*)(vy);
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
if (row >= nrows) {
return;
}
if (expert < 0) {
if (threadIdx.x == 0) {
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
}
return;
}
const int blocks_per_row = ncols / qk;
const int blocks_per_warp = vdr * WARP_SIZE / qi;
@@ -383,14 +383,17 @@ __global__ void Marlin(
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
int parallel = num_tokens_past_padded / moe_block_size;
int num_valid_blocks = parallel;
if constexpr (kIsEP) {
for (int i = 0; i < parallel; i++) {
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
}
// Every warp counts the same inactive blocks without a serial per-thread scan.
int num_invalid_blocks = 0;
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
num_invalid_blocks += expert_ids_ptr[i] == -1;
}
int num_invalid_blocks = parallel - num_valid_blocks;
parallel = num_valid_blocks;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
}
parallel -= num_invalid_blocks;
if (parallel == 0) return;
int k_tiles = prob_k / 16 / thread_k_blocks;
int n_tiles = prob_n / 16 / thread_n_blocks;
@@ -518,23 +521,18 @@ __global__ void Marlin(
if (par_id >= parallel) return;
old_expert_id = expert_id;
if constexpr (kIsEP) {
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
skip_count--;
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
}
} else {
block_id = par_id;
expert_id = expert_ids_ptr[block_id];
skip_count--;
};
}
} else {
block_id = par_id;
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
bool is_ep) {
int tid = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
int32_t neg_count = 0;
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
// Padding and nonlocal routes are both inactive, independently of EP.
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
for (int e = tid; e <= num_experts; e += stride) {
int32_t offset;
+10
View File
@@ -60,6 +60,16 @@ register_kernel(
)
register_kernel(
KernelSpec(
op="moe.shuffle_weighted_sum",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
capabilities=CUDA_ONLY,
)
)
def moe_align_block_size(
topk_ids: torch.Tensor,
num_experts: int,
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
use_int4_w4a16: tl.constexpr,
use_int8_w8a16: tl.constexpr,
even_Ks: tl.constexpr,
filter_expert: tl.constexpr,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
token_mask = offs_token < num_valid_tokens
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
if filter_expert and off_experts == -1:
# -----------------------------------------------------------
# Write back zeros to the output when the expert is not
# in the current expert parallel rank.
if off_experts == -1:
# Padding and nonlocal routes both use -1, including without EP.
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -377,12 +374,10 @@ def fused_moe_kernel(
per_channel_quant: tl.constexpr,
even_Ks: tl.constexpr,
c_sorted: tl.constexpr,
filter_expert: tl.constexpr,
swap_ab: tl.constexpr,
FUSE_ADD_TO_OUTPUT: tl.constexpr,
MASK_OUTPUT: tl.constexpr,
FUSE_SUM_ALL_REDUCE: tl.constexpr,
LORA_PRESERVE_BASE: tl.constexpr,
ROUTER_TOPK: tl.constexpr,
FUSE_SWIGLU: tl.constexpr = False,
USE_GDC: tl.constexpr = False,
@@ -450,16 +445,10 @@ def fused_moe_kernel(
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
off_experts = off_experts_i32.to(tl.int64)
if filter_expert and off_experts == -1:
if FUSE_SWIGLU:
# C is the half-width post-activation buffer here. Rows owned by a
# filtered expert are never read (the down-GEMM CTA for this block
# early-exits before loading A), and an N-wide zero store would run
# past the row end into a neighboring token's data.
return
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
# Write zeros only when this kernel owns the full output; the experimental LoRA
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
if off_experts == -1:
# The down GEMM never reads inactive sorted or half-width intermediates.
# Add/reduce outputs already hold their initial value and must stay intact.
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
lora_preserve_base: bool = False,
fuse_swiglu: bool = False,
) -> None:
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
assert topk_weights.stride(1) == 1
assert sorted_token_ids.stride(0) == 1
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
use_int4_w4a16=use_int4_w4a16,
use_int8_w8a16=use_int8_w8a16,
even_Ks=even_Ks,
filter_expert=filter_expert,
**config,
)
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
per_channel_quant=per_channel_quant,
even_Ks=even_Ks,
c_sorted=c_sorted,
filter_expert=filter_expert,
swap_ab=swap_ab,
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
MASK_OUTPUT=mask_output,
LORA_PRESERVE_BASE=lora_preserve_base,
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
ROUTER_TOPK=router_topk,
FUSE_SWIGLU=fuse_swiglu,
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
num_tokens,
stride_m,
has_expert_map: tl.constexpr,
is_ep: tl.constexpr,
has_topk_ids: tl.constexpr,
top_k: tl.constexpr,
size: tl.constexpr,
routed_scaling_factor: tl.constexpr,
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
if routed_scaling_factor != 1.0:
b_val = b_val * routed_scaling_factor
if has_expert_map:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
mapped_id = tl.load(
expert_map_ptr + tl.maximum(id_val, 0),
mask=m_mask & (id_val >= 0),
other=-1,
)
expert_mask = mapped_id >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
other=0.0,
).to(tl.float32)
elif is_ep:
elif has_topk_ids:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = id_val >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
Shape: (num_tokens, top_k).
outputs: Optional pre-allocated output tensor.
Shape: (num_tokens, hidden_size).
topk_ids: Optional indices of the top-k experts. Used when
`expert_map` is provided. Shape: (num_tokens, top_k).
topk_ids: Optional indices of the top-k experts. Negative ids always
skip the corresponding input load. Shape: (num_tokens, top_k).
expert_map: Optional mapping for Expert Parallelism. A value < 0
indicates an invalid token/expert pair that will be skipped.
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
# Keep is_ep as a compatible host argument; route validity follows ids.
num_tokens, top_k, size = inputs.shape
output_shape = (num_tokens, size)
if outputs is None:
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
num_tokens,
top_k * size,
expert_map is not None,
is_ep,
topk_ids is not None,
top_k,
size,
1.0 if routed_scaling_factor is None else routed_scaling_factor,
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
use_fp32_reduce: bool = False,
is_zp_float: bool = False,
) -> torch.Tensor:
"""Project routes, preserving inactive rows in a caller-provided output.
Inactive rows are zero when this function allocates the output.
"""
device = a.device
# Allocate output if not provided
if c_or_none is not None:
c = c_or_none
else:
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
# Early return for zero-size M
if size_m == 0:
@@ -0,0 +1,74 @@
"""Reduce routed rows while skipping inactive permutation entries."""
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
def _shuffle_weighted_sum_kernel(
inputs,
output,
permutation,
weights,
width: tl.constexpr,
rows,
topk: tl.constexpr,
input_stride: tl.constexpr,
output_stride: tl.constexpr,
weight_stride0,
weight_stride1,
BLOCK: tl.constexpr,
):
token = tl.program_id(0)
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
total = tl.full((BLOCK,), 0, tl.float32)
for slot in range(topk):
row = tl.load(permutation + token * topk + slot).to(tl.int64)
active = (row >= 0) & (row < rows)
values = tl.load(
inputs + row * input_stride + columns,
mask=active & (columns < width),
other=0,
).to(tl.float32)
factor = tl.load(
weights + token * weight_stride0 + slot * weight_stride1,
mask=active,
other=0,
).to(tl.float32)
total += values * factor
tl.store(
output + token.to(tl.int64) * output_stride + columns,
total,
mask=columns < width,
)
def shuffle_weighted_sum(
inputs: torch.Tensor,
output: torch.Tensor,
permutation: torch.Tensor,
weights: torch.Tensor,
) -> None:
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
assert inputs.ndim == output.ndim == weights.ndim == 2
assert inputs.shape[1] == output.shape[1]
assert inputs.stride(1) == output.stride(1) == 1
assert weights.shape[0] == output.shape[0]
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
if output.numel() == 0:
return
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
inputs,
output,
permutation,
weights,
output.shape[1],
inputs.shape[0],
weights.shape[1],
inputs.stride(0),
output.stride(0),
weights.stride(0),
weights.stride(1),
BLOCK=256,
)