fuse shared experts with routed experts in MegaMoE's DeepGEMM (#39313)

Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
Carrie Chen
2026-09-29 17:25:10 +08:00
committed by GitHub
co-authored by Brayden Zhong
parent c9cb32e310
commit 0e586fd12d
8 changed files with 160 additions and 21 deletions
@@ -31,8 +31,11 @@ struct MegaMoEPreDispatchParams {
int32_t* __restrict__ buf_x_sf; // row-major int32 [P, G/4]
int64_t* __restrict__ buf_topk_idx; // [padded_max, top_k]
float* __restrict__ buf_topk_weights; // [padded_max, top_k]
int32_t* __restrict__ shared_x_sf; // column-major, BLOCK_M-paged scales
uint64_t buf_x_sf_stride_bytes;
uint64_t shared_x_sf_col_stride;
uint32_t shared_block_m;
uint32_t num_tokens;
uint32_t padded_max;
uint32_t hidden;
@@ -44,7 +47,7 @@ struct MegaMoEPreDispatchParams {
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
// parameter rather than a runtime check because this kernel is issue-bound: the
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk>
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF>
__global__ __launch_bounds__(kMaxBlockThreads, 2) void //
mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) {
using namespace device;
@@ -66,6 +69,12 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
const uint32_t token_id = bid;
const auto token_in = params.x + static_cast<uint64_t>(token_id) * params.hidden;
const auto token_out = params.buf_x + static_cast<uint64_t>(token_id) * params.hidden;
uint32_t shared_row = 0;
if constexpr (kWriteSharedSF) {
const uint32_t m = token_id % params.shared_block_m;
shared_row = token_id / params.shared_block_m * ((params.shared_block_m + 127u) / 128u * 128u) + m / 128u * 128u +
m % 32u * 4u + m % 128u / 32u;
}
const auto quantize_chunk = [&](uint32_t chunk) {
InputVec in_vec;
@@ -104,6 +113,11 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
if (within_group_id == 0 && group_id < params.num_groups) {
const uint64_t byte_off = static_cast<uint64_t>(token_id) * params.buf_x_sf_stride_bytes + group_id;
reinterpret_cast<uint8_t*>(params.buf_x_sf)[byte_off] = static_cast<uint8_t>(ue8m0_exp);
if constexpr (kWriteSharedSF) {
const uint64_t off = shared_row + static_cast<uint64_t>(group_id / 4u) * params.shared_x_sf_col_stride;
reinterpret_cast<uint8_t*>(params.shared_x_sf)[off * sizeof(int32_t) + group_id % 4u] =
static_cast<uint8_t>(ue8m0_exp);
}
}
};
@@ -144,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
// ---- Host wrapper
// ------------------------------------------------------------------------------------------------------------------------
template <int64_t kGroupSize, bool kUsePDL>
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF>
struct MegaMoEPreDispatchKernel {
static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size");
static constexpr auto kernel_one_chunk =
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false>;
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF>;
static constexpr auto kernel_multi_chunk =
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true>;
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF>;
static void
run(const tvm::ffi::TensorView x,
@@ -159,7 +173,9 @@ struct MegaMoEPreDispatchKernel {
const tvm::ffi::TensorView buf_x,
const tvm::ffi::TensorView buf_x_sf,
const tvm::ffi::TensorView buf_topk_idx,
const tvm::ffi::TensorView buf_topk_weights) {
const tvm::ffi::TensorView buf_topk_weights,
const tvm::ffi::Optional<tvm::ffi::TensorView> shared_x_sf,
const int64_t shared_block_m) {
using namespace host;
auto device = SymbolicDevice{};
@@ -232,6 +248,21 @@ struct MegaMoEPreDispatchKernel {
"hidden above 8192 must be a multiple of 256 so the tail quant chunks fill whole warps");
RuntimeCheck(block_size >= top_k, "top_k must fit into one quant CTA");
int32_t* shared_sf_ptr = nullptr;
uint64_t shared_sf_stride = 0;
if constexpr (kWriteSharedSF) {
RuntimeCheck(kGroupSize == 32, "shared MegaMoE scales require group_size=32");
RuntimeCheck(shared_block_m > 0, "shared_block_m must be positive");
auto S = SymbolicSize{"shared_sf_rows"};
const auto sf = shared_x_sf.value();
TensorMatcher({S, G4}).with_strides({1, -1}).with_dtype<int32_t>().with_device(device).verify(sf);
const auto rows = (num_tokens + shared_block_m - 1) / shared_block_m * ((shared_block_m + 127) / 128 * 128);
RuntimeCheck(S.unwrap() >= rows, "shared scale buffer is too small for shared_block_m");
RuntimeCheck(sf.stride(1) >= S.unwrap(), "shared scale columns must not overlap");
shared_sf_ptr = static_cast<int32_t*>(sf.data_ptr());
shared_sf_stride = sf.stride(1);
}
const auto pad_slots = (padded_max - num_tokens) * top_k;
const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size);
const auto num_total_blocks = num_tokens + num_pad_blocks;
@@ -244,7 +275,10 @@ struct MegaMoEPreDispatchKernel {
.buf_x_sf = static_cast<int32_t*>(buf_x_sf.data_ptr()),
.buf_topk_idx = static_cast<int64_t*>(buf_topk_idx.data_ptr()),
.buf_topk_weights = static_cast<float*>(buf_topk_weights.data_ptr()),
.shared_x_sf = shared_sf_ptr,
.buf_x_sf_stride_bytes = buf_x_sf_stride_bytes,
.shared_x_sf_col_stride = shared_sf_stride,
.shared_block_m = static_cast<uint32_t>(shared_block_m),
.num_tokens = num_tokens,
.padded_max = padded_max,
.hidden = hidden,
+11 -3
View File
@@ -39,8 +39,8 @@ def _jit_hash_topk_module():
@cache_once
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int):
args = make_cpp_args(quant_group_size, is_arch_support_pdl())
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool):
args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf)
return load_jit(
make_name("mega_moe_pre_dispatch"),
*args,
@@ -162,8 +162,14 @@ def mega_moe_pre_dispatch(
buf_topk_idx: torch.Tensor,
buf_topk_weights: torch.Tensor,
quant_group_size: int = 32,
*,
shared_x_sf: Optional[torch.Tensor] = None,
shared_block_m: int = 0,
) -> None:
module = _jit_mega_moe_pre_dispatch_module(quant_group_size)
# shared_block_m must match the consuming MoE's paged scale layout
module = _jit_mega_moe_pre_dispatch_module(
quant_group_size, shared_x_sf is not None
)
module.run(
x,
topk_idx,
@@ -172,6 +178,8 @@ def mega_moe_pre_dispatch(
buf_x_sf,
buf_topk_idx,
buf_topk_weights,
shared_x_sf,
shared_block_m,
)
+1
View File
@@ -1288,6 +1288,7 @@ class Envs:
# Blackwell MegaMoE uses a whole-grid software barrier. Keep a small
# residency margin so every cluster can launch beside other streams.
SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2)
SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True)
# ===================================================================
# Top-k kernels
+88 -13
View File
@@ -34,7 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.models.deepseek_common.utils import _device_sm
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_hip
from sglang.srt.utils import is_hip, is_sm100_supported
if TYPE_CHECKING:
from deep_gemm import SymmBuffer
@@ -117,6 +117,7 @@ def _get_mega_moe_symm_buffer(
num_topk: int,
hidden: int,
intermediate_hidden: int,
num_shared_experts: int = 0,
mma_type: Optional[str] = None,
) -> SymmBuffer:
import deep_gemm
@@ -132,6 +133,7 @@ def _get_mega_moe_symm_buffer(
hidden,
intermediate_hidden,
mma_type,
num_shared_experts,
deep_gemm.get_num_sms(),
)
buf = _MEGA_MOE_SYMM_BUFFER.get(key)
@@ -143,6 +145,7 @@ def _get_mega_moe_symm_buffer(
num_topk,
hidden,
intermediate_hidden,
num_shared_experts=num_shared_experts,
mma_type=mma_type,
activation="swiglu",
)
@@ -183,6 +186,19 @@ def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool
return max_tokens_per_rank <= cap
def should_fuse_mega_moe_shared_experts(moe: DeepseekV2MoE) -> bool:
# DeepGEMM's in-kernel shared expert takes FP8 weights on SM100 fp8xfp4 only.
return (
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS.get()
and get_moe_a2a_backend().is_megamoe()
and getattr(moe.experts, "_mega_moe_weights_built", False)
and is_sm100_supported()
and _mega_moe_mma_type(moe.experts) == "fp8xfp4"
and moe.shared_experts_is_fp8
and moe.shared_experts_weight_block_size == [128, 128]
)
def forward_mega_moe(
moe: DeepseekV2MoE,
hidden_states: torch.Tensor,
@@ -199,29 +215,29 @@ def forward_mega_moe(
)
num_tokens = hidden_states.shape[0]
sbo_overlap_flag = (
fused_shared = moe.mega_shared_l1_weights is not None
fork_mega_moe = (
moe.alt_stream is not None
and moe.num_fused_shared_experts == 0
and (fused_shared or moe.num_fused_shared_experts == 0)
and num_tokens > 0
and get_is_capture_mode()
)
if sbo_overlap_flag:
if fork_mega_moe:
current_stream = torch.cuda.current_stream()
moe.alt_stream.wait_stream(current_stream)
shared_output = moe._forward_shared_experts(hidden_states)
mega_stream_ctx = torch.cuda.stream(moe.alt_stream)
else:
shared_output = moe._forward_shared_experts(hidden_states)
mega_stream_ctx = nullcontext()
shared_output = None if fused_shared else moe._forward_shared_experts(hidden_states)
with mega_stream_ctx:
y = _run_mega_routed(
moe, hidden_states, forward_batch, input_ids_global, num_tokens
)
if sbo_overlap_flag:
if fork_mega_moe:
current_stream.wait_stream(moe.alt_stream)
if shared_output is not None:
@@ -275,6 +291,11 @@ def _run_mega_routed(
if moe.experts.should_fuse_routed_scaling_factor_in_topk
else float(moe.routed_scaling_factor)
),
num_shared_experts=(
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
),
shared_l1_weights=moe.mega_shared_l1_weights,
shared_l2_weights=moe.mega_shared_l2_weights,
)
@@ -290,6 +311,9 @@ def run_mega_routed_experts(
num_tokens: int,
activation_clamp: Optional[float] = None,
routed_scaling_factor: float = 1.0,
num_shared_experts: int = 0,
shared_l1_weights=None,
shared_l2_weights=None,
) -> torch.Tensor:
# Rows are this rank's tokens; the returned rows are fully combined.
import deep_gemm
@@ -316,6 +340,7 @@ def run_mega_routed_experts(
num_topk=top_k,
hidden=hidden_size,
intermediate_hidden=intermediate_size,
num_shared_experts=num_shared_experts,
mma_type=mma_type,
)
@@ -339,6 +364,9 @@ def run_mega_routed_experts(
routed_scaling_factor=routed_scaling_factor,
)
if routed_scaling_factor != 1.0:
topk_weights_in = topk_weights_in * routed_scaling_factor
mega_kwargs = {"recipe": (1, 1, 32)}
if mma_type == "nvfp4xnvfp4":
# Per-token outer scales go to buf.x_scales; the kernel folds them with
@@ -380,6 +408,18 @@ def run_mega_routed_experts(
mma_type=mma_type,
)
else:
shared_block_m = 0
if shared_l1_weights is not None:
shared_block_m = deep_gemm.get_block_m_for_mega_moe(
num_ranks=buf.group.size(),
num_experts=buf.num_experts,
# DeepGEMM rounds the requested capacity up for its buffer.
num_max_tokens_per_rank=buf.num_max_tokens_per_rank,
# Match the non-null output allocation passed to DeepGEMM.
num_tokens=max(num_tokens, 1),
num_topk=buf.num_topk,
mma_type=buf.mma_type,
)
mega_moe_pre_dispatch(
hidden_states,
topk_ids_in,
@@ -389,6 +429,10 @@ def run_mega_routed_experts(
buf.topk_idx,
buf.topk_weights,
quant_group_size=32,
shared_x_sf=(
buf.shared_l1_acts_sf if shared_l1_weights is not None else None
),
shared_block_m=shared_block_m,
)
# Allocate at least one row so y has a non-null CUDA data_ptr;
@@ -404,16 +448,14 @@ def run_mega_routed_experts(
experts.mega_l1_weights,
experts.mega_l2_weights,
buf,
shared_l1_weights=shared_l1_weights,
shared_l2_weights=shared_l2_weights,
activation="swiglu",
activation_clamp=activation_clamp,
fast_math=True,
**mega_kwargs,
)
y = y[:num_tokens]
if routed_scaling_factor != 1.0:
y.mul_(routed_scaling_factor)
return y
return y[:num_tokens]
def _interleave_mega_moe_gate_up(t: torch.Tensor, gran: int = 8) -> torch.Tensor:
@@ -522,3 +564,36 @@ def build_mega_moe_experts_weights(experts) -> None:
experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp)
experts._mega_moe_weights_built = True
def build_mega_moe_shared_weights(
shared_experts,
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
from deep_gemm import transform_weights_for_mega_moe
gate_up_proj, down_proj = shared_experts.gate_up_proj, shared_experts.down_proj
# L1 is copied interleaved (2*moe_intermediate*hidden FP8 bytes per layer);
# the original stays for the forward_normal fallback above the token cap.
return transform_weights_for_mega_moe(
(gate_up_proj.weight.data, _transform_mega_moe_shared_sf(gate_up_proj)),
(down_proj.weight.data, _transform_mega_moe_shared_sf(down_proj)),
)
def _transform_mega_moe_shared_sf(linear) -> torch.Tensor:
from deep_gemm import transform_sf_into_required_layout
from sglang.srt.layers.quantization.fp8_utils import inverse_transform_scale_ue8m0
n, k = linear.weight.shape
sf = linear.weight_scale_inv.data
if sf.dtype == torch.int32:
# Requantized for the DeepGEMM linear runner.
sf = inverse_transform_scale_ue8m0(sf, mn=n)[:, : k // 128]
return transform_sf_into_required_layout(
sf.repeat_interleave(4, dim=-1),
mn=n,
k=k,
recipe=(128, 32),
disable_ue8m0_cast=False,
)
+2
View File
@@ -738,6 +738,8 @@ class DeepseekV2MoE(nn.Module):
self.shared_experts_is_fp8 = False
self.shared_experts_weight_block_size = None
self._shared_expert_tp1 = False
self.mega_shared_l1_weights = None
self.mega_shared_l2_weights = None
# Shared experts: skip when fused into MoE kernel
# (self.num_fused_shared_experts > 0) or when DeepEP/MegaMOE fusion is enabled.
if (
+14
View File
@@ -5219,6 +5219,20 @@ class DeepseekV4ForCausalLM(nn.Module):
self_attn.indexer.compressor.apply_ape_hotfix()
layer.refresh_mhc_norm_weight_cache()
def precompile_kernels_after_loading(self) -> None:
from sglang.srt.layers.moe.mega_moe import (
build_mega_moe_shared_weights,
should_fuse_mega_moe_shared_experts,
)
for module in self.modules():
if isinstance(
module, deepseek_v2.DeepseekV2MoE
) and should_fuse_mega_moe_shared_experts(module):
module.mega_shared_l1_weights, module.mega_shared_l2_weights = (
build_mega_moe_shared_weights(module.shared_experts)
)
@staticmethod
def remap_weight_name_to_dpsk_hf_format(
name: str,
@@ -783,6 +783,9 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
# embedding/head weights. The native CUDA path keeps the original DSpark
# behavior and shares the target model's vocabulary modules.
uses_own_vocab_modules = _is_npu
precompile_kernels_after_loading = (
DeepseekV4ForCausalLM.precompile_kernels_after_loading
)
@classmethod
def shared_experts_fusion_disable_reason(