[Perf] dsv41: Integrate DeepGEMM MegaGate routing (#43262)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-10-22 19:08:39 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c82f7141ad
commit e5dc0c363a
12 changed files with 779 additions and 34 deletions
@@ -47,7 +47,7 @@ struct MegaMoEPreDispatchParams {
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
// parameter rather than a runtime check because this kernel is issue-bound: the
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF>
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF, bool kCopyTopk>
__global__ __launch_bounds__(kMaxBlockThreads, 2) void //
mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) {
using namespace device;
@@ -135,7 +135,7 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
}
// Copy this token's topk row (no alignment assumptions; top_k is small).
if (tid < params.top_k) {
if (kCopyTopk && tid < params.top_k) {
const uint32_t off = token_id * params.top_k + tid;
params.buf_topk_idx[off] = params.topk_idx[off];
params.buf_topk_weights[off] = params.topk_weights[off];
@@ -158,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
// ---- Host wrapper
// ------------------------------------------------------------------------------------------------------------------------
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF>
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF, bool kCopyTopk>
struct MegaMoEPreDispatchKernel {
static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size");
static constexpr auto kernel_one_chunk =
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF>;
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF, kCopyTopk>;
static constexpr auto kernel_multi_chunk =
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF>;
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF, kCopyTopk>;
static void
run(const tvm::ffi::TensorView x,
@@ -190,14 +190,16 @@ struct MegaMoEPreDispatchKernel {
.with_dtype<bf16_t>()
.with_device(device)
.verify(x);
TensorMatcher({M, K}) // topk_idx
.with_dtype<int32_t>()
.with_device(device)
.verify(topk_idx);
TensorMatcher({M, K}) // topk_weights
.with_dtype<float>()
.with_device(device)
.verify(topk_weights);
if constexpr (kCopyTopk) {
TensorMatcher({M, K}) // topk_idx
.with_dtype<int32_t>()
.with_device(device)
.verify(topk_idx);
TensorMatcher({M, K}) // topk_weights
.with_dtype<float>()
.with_device(device)
.verify(topk_weights);
}
// DeepGEMM versions expose this fp8 dispatch buffer either as raw int8
// storage or as torch.float8_e4m3fn; the kernel writes fp8 bytes in both.
TensorMatcher({P, H}) // buf.x
@@ -263,7 +265,7 @@ struct MegaMoEPreDispatchKernel {
shared_sf_stride = sf.stride(1);
}
const auto pad_slots = (padded_max - num_tokens) * top_k;
const auto pad_slots = kCopyTopk ? (padded_max - num_tokens) * top_k : 0u;
const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size);
const auto num_total_blocks = num_tokens + num_pad_blocks;
+11
View File
@@ -27,6 +27,17 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="moe.bf16_mega_gate",
backend=KernelBackend.DEEPGEMM,
target="sglang.kernels.ops.moe.mega_gate:bf16_mega_gate",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
),
)
)
register_kernel(
KernelSpec(
op="moe.moe_align_block_size",
+8 -3
View File
@@ -36,8 +36,12 @@ def _jit_hash_topk_module():
@cache_once
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool):
args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf)
def _jit_mega_moe_pre_dispatch_module(
quant_group_size: int, write_shared_sf: bool, copy_topk: bool = True
):
args = make_cpp_args(
quant_group_size, is_arch_support_pdl(), write_shared_sf, copy_topk
)
return load_jit(
make_name("mega_moe_pre_dispatch"),
*args,
@@ -127,12 +131,13 @@ def mega_moe_pre_dispatch(
*,
shared_x_sf: Optional[torch.Tensor] = None,
shared_block_m: int = 0,
copy_topk: bool = True,
) -> None:
if x.shape[0] == 0 and x.stride(0) != x.shape[1]:
x = x.new_empty(x.shape)
# shared_block_m must match the consuming MoE's paged scale layout
module = _jit_mega_moe_pre_dispatch_module(
quant_group_size, shared_x_sf is not None
quant_group_size, shared_x_sf is not None, copy_topk
)
module.run(
x,
@@ -0,0 +1,61 @@
from __future__ import annotations
from typing import Optional
import torch
def bf16_mega_gate(
hidden_states: torch.Tensor,
weight: torch.Tensor,
top_k: int,
*,
routed_scaling_factor: float,
ep_rank: int,
bias: Optional[torch.Tensor] = None,
image_bias: Optional[torch.Tensor] = None,
image_token_mask: Optional[torch.Tensor] = None,
valid_token_mask: Optional[torch.Tensor] = None,
hash_topk_ids: Optional[torch.Tensor] = None,
to_physical_map: Optional[torch.Tensor] = None,
logical_count: Optional[torch.Tensor] = None,
out: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
import deep_gemm
if valid_token_mask is not None:
hidden_states = torch.where(valid_token_mask[:, None], hidden_states, 0)
if out is None:
ids = torch.empty(
(hidden_states.shape[0], top_k),
dtype=torch.int64,
device=hidden_states.device,
)
weights = torch.empty_like(ids, dtype=torch.float32)
else:
ids, weights = out
deep_gemm.bf16_mega_gate(
hidden_states,
weight,
top_k,
use_shared_as_routed=False,
num_shared_experts=0,
routed_scaling_factor=routed_scaling_factor,
ep_rank=ep_rank,
scoring_func="sqrtsoftplus",
mask=valid_token_mask,
bias=bias,
image_bias=image_bias,
image_token_mask=image_token_mask,
fix_routing_mask=(
torch.ones_like(ids[:, 0], dtype=torch.bool)
if hash_topk_ids is not None
else None
),
unmapped_topk_idx=hash_topk_ids,
to_physical_map=to_physical_map,
logical_count=logical_count,
out=(ids, weights),
)
return weights, ids
+1
View File
@@ -1432,6 +1432,7 @@ class Envs:
# residency margin so every cluster can launch beside other streams.
SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2)
SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True)
SGLANG_OPT_DEEPGEMM_MEGA_GATE = EnvBool(False)
# ===================================================================
# Top-k kernels
+221
View File
@@ -0,0 +1,221 @@
from __future__ import annotations
import functools
from typing import NamedTuple, Optional
import torch
from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.moe.topk import (
TopKConfig,
_post_process_topk_ids,
apply_routing_simulation,
routing_simulation_enabled,
)
from sglang.srt.models.deepseek_common.utils import _device_sm
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
# DeepGEMM bf16_mega_gate scores one token block per launch.
_MEGA_GATE_MAX_ROWS = 2048
class MegaGateRouting(NamedTuple):
topk_weights: Optional[torch.Tensor]
topk_ids: Optional[torch.Tensor]
routed_scaling_factor: float
in_buffer: bool
def mega_gate_supported(moe) -> bool:
if (
_device_sm // 10 != 10
or moe.config.model_type != "deepseek_v41"
or moe.topk.enable_waterfill
or moe.num_fused_shared_experts
or get_exec().deterministic.enable_deterministic_inference
):
return False
weight = moe.gate.weight
num_experts, hidden = weight.shape
top_k = moe.config.num_experts_per_tok
if (
weight.dtype != torch.bfloat16
or not weight.is_contiguous()
or hidden % 256 != 0
or not 0 < num_experts <= 512
or num_experts % 4 != 0
or not 0 < top_k <= min(32, num_experts)
):
return False
bias_vl = None if moe.is_hash else moe.gate.e_score_correction_bias_vl
for bias in (moe.gate.e_score_correction_bias, bias_vl):
if bias is not None and (
bias.dtype != torch.float32
or not bias.is_contiguous()
or bias.shape != (num_experts,)
):
return False
if moe.is_hash:
return (
moe.topk.score_func == "sqrtsoftplus"
and moe.topk.num_shared_topk_slots == 0
and moe.topk.topk == top_k
)
config = moe.topk.topk_config
return (
config.scoring_func == "sqrtsoftplus"
and config.sqrtsoftplus_log1p
and config.renormalize
and config.num_shared_topk_slots == 0
and config.top_k == top_k
and not (top_k == 1 and bias_vl is not None)
and not config.use_grouped_topk
and config.custom_routing_function is None
and not config.torch_native
)
def should_use_mega_gate(moe, hidden_states: torch.Tensor) -> bool:
forward = get_forward()
return (
envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get()
and moe.mega_gate_supported
and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS
and not forward.is_extend_in_batch
and not forward.sp_active
and hidden_states.dtype == torch.bfloat16
and hidden_states.is_contiguous()
)
def _captures_routed_experts(moe) -> bool:
return (
not moe.is_hash
and moe.topk.topk_config.allow_routed_experts_capture
and get_global_experts_capturer() is not None
)
@functools.lru_cache
def _ones(num_experts: int, device: torch.device) -> torch.Tensor:
return torch.ones(num_experts, dtype=torch.int32, device=device)
def _physical_dispatch_maps(info) -> tuple[torch.Tensor, torch.Tensor]:
if info.ep_dispatch_algorithm == "static":
rank_map = info.partial_logical_to_rank_dispatch_physical_map
return rank_map.to(torch.int32)[:, None], _ones(len(rank_map), rank_map.device)
return (
info.partial_logical_to_all_physical_map.to(torch.int32),
info.partial_logical_to_all_physical_map_num_valid.to(torch.int32),
)
def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateRouting]:
bound, dispatch_info = moe.topk.routing_inputs()
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm in (
"lp",
"fake",
):
return None
if moe.is_hash and input_ids is None:
return None
num_tokens = hidden_states.shape[0]
valid = None
if bound is not None:
valid = (
torch.arange(num_tokens, device=hidden_states.device, dtype=torch.int32)
< bound.reshape(-1)[:1]
)
image = hash_ids = None
if moe.is_hash:
token_ids = input_ids[:num_tokens]
if valid is not None:
token_ids = torch.where(valid, token_ids, 0)
hash_ids = moe.topk.tid2eid[token_ids].to(torch.int64)
elif moe.gate.e_score_correction_bias_vl is not None and input_ids is not None:
image = (input_ids[:num_tokens] == moe.config.image_token_id).contiguous()
routed_scaling_factor = float(moe.routed_scaling_factor)
fused_in_topk = moe.routed_scaling_factor_fused_in_topk
# The benchmark override writes logical ids, so with EPLB it must run before the remap.
in_buffer = (
buf is not None
and not _captures_routed_experts(moe)
and not (dispatch_info is not None and routing_simulation_enabled())
)
to_physical_map = logical_count = None
if in_buffer and dispatch_info is not None:
# NOTE(kpham-sgl): MegaMoE reads physical expert slots from this buffer, so
# under EPLB the gate must map logical ids in-kernel to stay on this path.
to_physical_map, logical_count = _physical_dispatch_maps(dispatch_info)
weights, ids = bf16_mega_gate(
hidden_states,
moe.gate.weight,
moe.config.num_experts_per_tok,
routed_scaling_factor=(
routed_scaling_factor if in_buffer or fused_in_topk else 1.0
),
ep_rank=get_parallel().moe_ep_rank,
bias=moe.gate.e_score_correction_bias,
image_bias=moe.gate.e_score_correction_bias_vl if image is not None else None,
image_token_mask=image,
valid_token_mask=valid,
hash_topk_ids=hash_ids,
to_physical_map=to_physical_map,
logical_count=logical_count,
out=(
(buf.topk_idx[:num_tokens], buf.topk_weights[:num_tokens])
if in_buffer
else None
),
)
if not moe.is_hash and routing_simulation_enabled():
top_k = ids.shape[1]
apply_routing_simulation(
ids,
weights,
moe.gate.weight.shape[0],
num_routed_topk=top_k,
top_k=top_k,
layer_id=moe.layer_id,
num_token_non_padded=bound,
weight_scale=(
routed_scaling_factor if in_buffer and not fused_in_topk else 1.0
),
)
if in_buffer:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=buf.topk_idx[:num_tokens]
)
return MegaGateRouting(None, None, 1.0, True)
if moe.is_hash:
config = TopKConfig(
top_k=moe.topk.topk,
routed_scaling_factor=moe.topk.routed_scaling_factor,
apply_routed_scaling_factor_on_output=moe.topk.apply_routed_scaling_factor_on_output,
allow_routed_experts_capture=False,
)
else:
config = moe.topk.topk_config
ids, weights, recorder_ids = _post_process_topk_ids(
ids,
weights,
config,
hidden_states.new_empty((0, moe.gate.weight.shape[0])),
moe.layer_id,
num_token_non_padded=bound,
expert_location_dispatch_info=dispatch_info,
padded_rows_masked=True,
)
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids
)
return MegaGateRouting(
weights, ids, 1.0 if fused_in_topk else routed_scaling_factor, False
)
+55 -9
View File
@@ -25,6 +25,7 @@ from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
from sglang.srt.environ import envs
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
from sglang.srt.layers.moe.mega_gate import run_mega_gate, should_use_mega_gate
from sglang.srt.layers.moe.mega_moe_sm90 import (
is_sm90_fp8_mega_moe_available,
run_sm90_mega_routed,
@@ -250,6 +251,25 @@ def forward_mega_moe(
return y
def _mega_moe_buffer(moe: DeepseekV2MoE):
from sglang.srt.runtime_context import get_parallel
return _get_mega_moe_symm_buffer(
get_parallel().moe_ep_group.device_group,
num_experts=moe.experts.num_experts,
num_max_tokens_per_rank=(
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
),
num_topk=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
hidden=moe.config.hidden_size,
intermediate_hidden=moe.config.moe_intermediate_size,
num_shared_experts=(
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
),
mma_type=_mega_moe_mma_type(moe.experts),
)
def _run_mega_routed(
moe: DeepseekV2MoE,
hidden_states: torch.Tensor,
@@ -259,7 +279,24 @@ def _run_mega_routed(
) -> torch.Tensor:
hidden_size = moe.config.hidden_size
if num_tokens > 0:
routing = None
if should_use_mega_gate(moe, hidden_states):
import deep_gemm
# Only the fp8xfp4 pre-dispatch can skip its top-k copy (kCopyTopk).
buf = (
_mega_moe_buffer(moe)
if _mega_moe_mma_type(moe.experts) == "fp8xfp4"
else None
)
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
routing = run_mega_gate(moe, hidden_states, input_ids_global, buf)
if routing is not None:
topk_ids = routing.topk_ids
topk_weights = routing.topk_weights
routed_scaling_factor = routing.routed_scaling_factor
elif num_tokens > 0:
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
if (
@@ -282,6 +319,12 @@ def _run_mega_routed(
else:
topk_ids = None
topk_weights = None
if routing is None:
routed_scaling_factor = (
1.0
if moe.routed_scaling_factor_fused_in_topk
else float(moe.routed_scaling_factor)
)
return run_mega_routed_experts(
moe.experts,
@@ -293,16 +336,13 @@ def _run_mega_routed(
top_k=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
num_tokens=num_tokens,
activation_clamp=moe.experts.moe_runner_config.swiglu_limit,
routed_scaling_factor=(
1.0
if moe.routed_scaling_factor_fused_in_topk
else float(moe.routed_scaling_factor)
),
routed_scaling_factor=routed_scaling_factor,
num_shared_experts=(
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
),
shared_l1_weights=moe.mega_shared_l1_weights,
shared_l2_weights=moe.mega_shared_l2_weights,
routing_in_buffer=routing is not None and routing.in_buffer,
)
@@ -321,6 +361,7 @@ def run_mega_routed_experts(
num_shared_experts: int = 0,
shared_l1_weights=None,
shared_l2_weights=None,
routing_in_buffer: bool = False,
) -> torch.Tensor:
# Rows are this rank's tokens; the returned rows are fully combined.
import deep_gemm
@@ -351,7 +392,10 @@ def run_mega_routed_experts(
mma_type=mma_type,
)
if num_tokens > 0:
if routing_in_buffer:
assert num_tokens > 0 and _device_sm != 90 and mma_type == "fp8xfp4"
topk_ids_in = topk_weights_in = None
elif num_tokens > 0:
topk_ids_in = topk_ids.to(torch.int32)
topk_weights_in = topk_weights.to(torch.float32)
else:
@@ -372,6 +416,7 @@ def run_mega_routed_experts(
)
if routed_scaling_factor != 1.0:
assert not routing_in_buffer
topk_weights_in = topk_weights_in * routed_scaling_factor
mega_kwargs = {"recipe": (1, 1, 32)}
@@ -429,8 +474,8 @@ def run_mega_routed_experts(
)
mega_moe_pre_dispatch(
hidden_states,
topk_ids_in,
topk_weights_in,
buf.topk_idx if routing_in_buffer else topk_ids_in,
buf.topk_weights if routing_in_buffer else topk_weights_in,
buf.x,
buf.x_sf,
buf.topk_idx,
@@ -440,6 +485,7 @@ def run_mega_routed_experts(
buf.shared_l1_acts_sf if shared_l1_weights is not None else None
),
shared_block_m=shared_block_m,
copy_topk=not routing_in_buffer,
)
# Allocate at least one row so y has a non-null CUDA data_ptr;
+17 -6
View File
@@ -467,9 +467,11 @@ def _simulate_balanced_routing_kernel(
stride_ik,
stride_wm,
stride_wk,
num_non_padded_ptr,
K: tl.constexpr,
BLOCK_K: tl.constexpr,
RANDOM: tl.constexpr,
HAS_NON_PADDED: tl.constexpr = False,
):
"""One program per token: overwrite its top-k row with a balanced expert
assignment and uniform ``1/k`` weights, in a single launch — so the
@@ -505,12 +507,13 @@ def _simulate_balanced_routing_kernel(
gid = (deal % num_ranks) * experts_per_rank + (
deal // num_ranks
) % experts_per_rank
w = tl.full((BLOCK_K,), inv_k, tl.float32)
if HAS_NON_PADDED:
is_pad = t >= tl.load(num_non_padded_ptr)
gid = tl.where(is_pad, -1, gid)
w = tl.where(is_pad, 0.0, w)
tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask)
tl.store(
topk_weights_ptr + t * stride_wm + j * stride_wk,
tl.full((BLOCK_K,), inv_k, tl.float32),
mask=mask,
)
tl.store(topk_weights_ptr + t * stride_wm + j * stride_wk, w, mask=mask)
# Per-launch seed for the uniform (RANDOM=True) path: varies across eager calls so
@@ -529,6 +532,8 @@ def _simulate_balanced_routing(
token_shard_rank: int = 0,
num_token_shards: int = 1,
seed: Optional[int] = None,
num_token_non_padded: Optional[torch.Tensor] = None,
weight_scale: float = 1.0,
) -> None:
"""Benchmark-only fused override (in place): replace ``topk_ids`` with a
balanced expert assignment and ``topk_weights`` with ``1/k`` using a single
@@ -589,7 +594,7 @@ def _simulate_balanced_routing(
num_ranks,
experts_per_rank,
deal_offset,
1.0 / k,
weight_scale / k,
seed,
token_shard_rank,
num_token_shards,
@@ -597,9 +602,11 @@ def _simulate_balanced_routing(
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
num_token_non_padded if num_token_non_padded is not None else topk_ids,
K=k,
BLOCK_K=triton.next_power_of_2(k),
RANDOM=random,
HAS_NON_PADDED=num_token_non_padded is not None,
)
@@ -2401,6 +2408,8 @@ def apply_routing_simulation(
num_routed_topk: int,
top_k: int,
layer_id: Optional[int],
num_token_non_padded: Optional[torch.Tensor] = None,
weight_scale: float = 1.0,
) -> None:
simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
# Benchmark-only: override gating with a balanced expert assignment (so
@@ -2441,6 +2450,8 @@ def apply_routing_simulation(
layer_id=layer_id,
token_shard_rank=token_shard_rank,
num_token_shards=num_token_shards,
num_token_non_padded=num_token_non_padded,
weight_scale=weight_scale,
)
+6
View File
@@ -1869,6 +1869,12 @@ class DeepseekV2MoE(nn.Module):
def _routed_mxfp8_prequant_static_enabled(self) -> bool:
return self._compute_routed_mxfp8_prequant_enabled()[0]
@cached_property
def mega_gate_supported(self) -> bool:
from sglang.srt.layers.moe.mega_gate import mega_gate_supported
return mega_gate_supported(self)
def _compute_routed_mxfp8_prequant_enabled(self) -> Tuple[bool, str]:
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher