diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh index 203a96c5cf..84478333c1 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh @@ -47,7 +47,7 @@ struct MegaMoEPreDispatchParams { // kMultiChunk covers rows too wide for one chunk per thread. It is a template // parameter rather than a runtime check because this kernel is issue-bound: the // loop bookkeeping alone costs ~10% on the rows that fit a single chunk. -template +template __global__ __launch_bounds__(kMaxBlockThreads, 2) void // mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) { using namespace device; @@ -135,7 +135,7 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void // } // Copy this token's topk row (no alignment assumptions; top_k is small). - if (tid < params.top_k) { + if (kCopyTopk && tid < params.top_k) { const uint32_t off = token_id * params.top_k + tid; params.buf_topk_idx[off] = params.topk_idx[off]; params.buf_topk_weights[off] = params.topk_weights[off]; @@ -158,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void // // ---- Host wrapper // ------------------------------------------------------------------------------------------------------------------------ -template +template struct MegaMoEPreDispatchKernel { static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size"); static constexpr auto kernel_one_chunk = - mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, false, kWriteSharedSF>; + mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, false, kWriteSharedSF, kCopyTopk>; static constexpr auto kernel_multi_chunk = - mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, true, kWriteSharedSF>; + mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, true, kWriteSharedSF, kCopyTopk>; static void run(const tvm::ffi::TensorView x, @@ -190,14 +190,16 @@ struct MegaMoEPreDispatchKernel { .with_dtype() .with_device(device) .verify(x); - TensorMatcher({M, K}) // topk_idx - .with_dtype() - .with_device(device) - .verify(topk_idx); - TensorMatcher({M, K}) // topk_weights - .with_dtype() - .with_device(device) - .verify(topk_weights); + if constexpr (kCopyTopk) { + TensorMatcher({M, K}) // topk_idx + .with_dtype() + .with_device(device) + .verify(topk_idx); + TensorMatcher({M, K}) // topk_weights + .with_dtype() + .with_device(device) + .verify(topk_weights); + } // DeepGEMM versions expose this fp8 dispatch buffer either as raw int8 // storage or as torch.float8_e4m3fn; the kernel writes fp8 bytes in both. TensorMatcher({P, H}) // buf.x @@ -263,7 +265,7 @@ struct MegaMoEPreDispatchKernel { shared_sf_stride = sf.stride(1); } - const auto pad_slots = (padded_max - num_tokens) * top_k; + const auto pad_slots = kCopyTopk ? (padded_max - num_tokens) * top_k : 0u; const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size); const auto num_total_blocks = num_tokens + num_pad_blocks; diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index b92ddfcb14..173b817c61 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -27,6 +27,17 @@ register_kernel( ) ) +register_kernel( + KernelSpec( + op="moe.bf16_mega_gate", + backend=KernelBackend.DEEPGEMM, + target="sglang.kernels.ops.moe.mega_gate:bf16_mega_gate", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} + ), + ) +) + register_kernel( KernelSpec( op="moe.moe_align_block_size", diff --git a/python/sglang/kernels/ops/moe/dsv4.py b/python/sglang/kernels/ops/moe/dsv4.py index d4d62535fc..e3ecc96cf3 100644 --- a/python/sglang/kernels/ops/moe/dsv4.py +++ b/python/sglang/kernels/ops/moe/dsv4.py @@ -36,8 +36,12 @@ def _jit_hash_topk_module(): @cache_once -def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool): - args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf) +def _jit_mega_moe_pre_dispatch_module( + quant_group_size: int, write_shared_sf: bool, copy_topk: bool = True +): + args = make_cpp_args( + quant_group_size, is_arch_support_pdl(), write_shared_sf, copy_topk + ) return load_jit( make_name("mega_moe_pre_dispatch"), *args, @@ -127,12 +131,13 @@ def mega_moe_pre_dispatch( *, shared_x_sf: Optional[torch.Tensor] = None, shared_block_m: int = 0, + copy_topk: bool = True, ) -> None: if x.shape[0] == 0 and x.stride(0) != x.shape[1]: x = x.new_empty(x.shape) # shared_block_m must match the consuming MoE's paged scale layout module = _jit_mega_moe_pre_dispatch_module( - quant_group_size, shared_x_sf is not None + quant_group_size, shared_x_sf is not None, copy_topk ) module.run( x, diff --git a/python/sglang/kernels/ops/moe/mega_gate.py b/python/sglang/kernels/ops/moe/mega_gate.py new file mode 100644 index 0000000000..c526454915 --- /dev/null +++ b/python/sglang/kernels/ops/moe/mega_gate.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Optional + +import torch + + +def bf16_mega_gate( + hidden_states: torch.Tensor, + weight: torch.Tensor, + top_k: int, + *, + routed_scaling_factor: float, + ep_rank: int, + bias: Optional[torch.Tensor] = None, + image_bias: Optional[torch.Tensor] = None, + image_token_mask: Optional[torch.Tensor] = None, + valid_token_mask: Optional[torch.Tensor] = None, + hash_topk_ids: Optional[torch.Tensor] = None, + to_physical_map: Optional[torch.Tensor] = None, + logical_count: Optional[torch.Tensor] = None, + out: Optional[tuple[torch.Tensor, torch.Tensor]] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + import deep_gemm + + if valid_token_mask is not None: + hidden_states = torch.where(valid_token_mask[:, None], hidden_states, 0) + + if out is None: + ids = torch.empty( + (hidden_states.shape[0], top_k), + dtype=torch.int64, + device=hidden_states.device, + ) + weights = torch.empty_like(ids, dtype=torch.float32) + else: + ids, weights = out + deep_gemm.bf16_mega_gate( + hidden_states, + weight, + top_k, + use_shared_as_routed=False, + num_shared_experts=0, + routed_scaling_factor=routed_scaling_factor, + ep_rank=ep_rank, + scoring_func="sqrtsoftplus", + mask=valid_token_mask, + bias=bias, + image_bias=image_bias, + image_token_mask=image_token_mask, + fix_routing_mask=( + torch.ones_like(ids[:, 0], dtype=torch.bool) + if hash_topk_ids is not None + else None + ), + unmapped_topk_idx=hash_topk_ids, + to_physical_map=to_physical_map, + logical_count=logical_count, + out=(ids, weights), + ) + return weights, ids diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c7cf2f1a82..3d4ff75513 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1432,6 +1432,7 @@ class Envs: # residency margin so every cluster can launch beside other streams. SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2) SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True) + SGLANG_OPT_DEEPGEMM_MEGA_GATE = EnvBool(False) # =================================================================== # Top-k kernels diff --git a/python/sglang/srt/layers/moe/mega_gate.py b/python/sglang/srt/layers/moe/mega_gate.py new file mode 100644 index 0000000000..192605e8a8 --- /dev/null +++ b/python/sglang/srt/layers/moe/mega_gate.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import functools +from typing import NamedTuple, Optional + +import torch + +from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate +from sglang.srt.environ import envs +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.moe.topk import ( + TopKConfig, + _post_process_topk_ids, + apply_routing_simulation, + routing_simulation_enabled, +) +from sglang.srt.models.deepseek_common.utils import _device_sm +from sglang.srt.runtime_context import get_exec, get_forward, get_parallel +from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer + +# DeepGEMM bf16_mega_gate scores one token block per launch. +_MEGA_GATE_MAX_ROWS = 2048 + + +class MegaGateRouting(NamedTuple): + topk_weights: Optional[torch.Tensor] + topk_ids: Optional[torch.Tensor] + routed_scaling_factor: float + in_buffer: bool + + +def mega_gate_supported(moe) -> bool: + if ( + _device_sm // 10 != 10 + or moe.config.model_type != "deepseek_v41" + or moe.topk.enable_waterfill + or moe.num_fused_shared_experts + or get_exec().deterministic.enable_deterministic_inference + ): + return False + weight = moe.gate.weight + num_experts, hidden = weight.shape + top_k = moe.config.num_experts_per_tok + if ( + weight.dtype != torch.bfloat16 + or not weight.is_contiguous() + or hidden % 256 != 0 + or not 0 < num_experts <= 512 + or num_experts % 4 != 0 + or not 0 < top_k <= min(32, num_experts) + ): + return False + bias_vl = None if moe.is_hash else moe.gate.e_score_correction_bias_vl + for bias in (moe.gate.e_score_correction_bias, bias_vl): + if bias is not None and ( + bias.dtype != torch.float32 + or not bias.is_contiguous() + or bias.shape != (num_experts,) + ): + return False + if moe.is_hash: + return ( + moe.topk.score_func == "sqrtsoftplus" + and moe.topk.num_shared_topk_slots == 0 + and moe.topk.topk == top_k + ) + config = moe.topk.topk_config + return ( + config.scoring_func == "sqrtsoftplus" + and config.sqrtsoftplus_log1p + and config.renormalize + and config.num_shared_topk_slots == 0 + and config.top_k == top_k + and not (top_k == 1 and bias_vl is not None) + and not config.use_grouped_topk + and config.custom_routing_function is None + and not config.torch_native + ) + + +def should_use_mega_gate(moe, hidden_states: torch.Tensor) -> bool: + forward = get_forward() + return ( + envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() + and moe.mega_gate_supported + and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS + and not forward.is_extend_in_batch + and not forward.sp_active + and hidden_states.dtype == torch.bfloat16 + and hidden_states.is_contiguous() + ) + + +def _captures_routed_experts(moe) -> bool: + return ( + not moe.is_hash + and moe.topk.topk_config.allow_routed_experts_capture + and get_global_experts_capturer() is not None + ) + + +@functools.lru_cache +def _ones(num_experts: int, device: torch.device) -> torch.Tensor: + return torch.ones(num_experts, dtype=torch.int32, device=device) + + +def _physical_dispatch_maps(info) -> tuple[torch.Tensor, torch.Tensor]: + if info.ep_dispatch_algorithm == "static": + rank_map = info.partial_logical_to_rank_dispatch_physical_map + return rank_map.to(torch.int32)[:, None], _ones(len(rank_map), rank_map.device) + return ( + info.partial_logical_to_all_physical_map.to(torch.int32), + info.partial_logical_to_all_physical_map_num_valid.to(torch.int32), + ) + + +def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateRouting]: + bound, dispatch_info = moe.topk.routing_inputs() + if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm in ( + "lp", + "fake", + ): + return None + if moe.is_hash and input_ids is None: + return None + num_tokens = hidden_states.shape[0] + valid = None + if bound is not None: + valid = ( + torch.arange(num_tokens, device=hidden_states.device, dtype=torch.int32) + < bound.reshape(-1)[:1] + ) + image = hash_ids = None + if moe.is_hash: + token_ids = input_ids[:num_tokens] + if valid is not None: + token_ids = torch.where(valid, token_ids, 0) + hash_ids = moe.topk.tid2eid[token_ids].to(torch.int64) + elif moe.gate.e_score_correction_bias_vl is not None and input_ids is not None: + image = (input_ids[:num_tokens] == moe.config.image_token_id).contiguous() + + routed_scaling_factor = float(moe.routed_scaling_factor) + fused_in_topk = moe.routed_scaling_factor_fused_in_topk + # The benchmark override writes logical ids, so with EPLB it must run before the remap. + in_buffer = ( + buf is not None + and not _captures_routed_experts(moe) + and not (dispatch_info is not None and routing_simulation_enabled()) + ) + to_physical_map = logical_count = None + if in_buffer and dispatch_info is not None: + # NOTE(kpham-sgl): MegaMoE reads physical expert slots from this buffer, so + # under EPLB the gate must map logical ids in-kernel to stay on this path. + to_physical_map, logical_count = _physical_dispatch_maps(dispatch_info) + weights, ids = bf16_mega_gate( + hidden_states, + moe.gate.weight, + moe.config.num_experts_per_tok, + routed_scaling_factor=( + routed_scaling_factor if in_buffer or fused_in_topk else 1.0 + ), + ep_rank=get_parallel().moe_ep_rank, + bias=moe.gate.e_score_correction_bias, + image_bias=moe.gate.e_score_correction_bias_vl if image is not None else None, + image_token_mask=image, + valid_token_mask=valid, + hash_topk_ids=hash_ids, + to_physical_map=to_physical_map, + logical_count=logical_count, + out=( + (buf.topk_idx[:num_tokens], buf.topk_weights[:num_tokens]) + if in_buffer + else None + ), + ) + if not moe.is_hash and routing_simulation_enabled(): + top_k = ids.shape[1] + apply_routing_simulation( + ids, + weights, + moe.gate.weight.shape[0], + num_routed_topk=top_k, + top_k=top_k, + layer_id=moe.layer_id, + num_token_non_padded=bound, + weight_scale=( + routed_scaling_factor if in_buffer and not fused_in_topk else 1.0 + ), + ) + if in_buffer: + get_global_expert_distribution_recorder().on_select_experts( + topk_ids=buf.topk_idx[:num_tokens] + ) + return MegaGateRouting(None, None, 1.0, True) + + if moe.is_hash: + config = TopKConfig( + top_k=moe.topk.topk, + routed_scaling_factor=moe.topk.routed_scaling_factor, + apply_routed_scaling_factor_on_output=moe.topk.apply_routed_scaling_factor_on_output, + allow_routed_experts_capture=False, + ) + else: + config = moe.topk.topk_config + ids, weights, recorder_ids = _post_process_topk_ids( + ids, + weights, + config, + hidden_states.new_empty((0, moe.gate.weight.shape[0])), + moe.layer_id, + num_token_non_padded=bound, + expert_location_dispatch_info=dispatch_info, + padded_rows_masked=True, + ) + if recorder_ids is not None: + get_global_expert_distribution_recorder().on_select_experts( + topk_ids=recorder_ids + ) + return MegaGateRouting( + weights, ids, 1.0 if fused_in_topk else routed_scaling_factor, False + ) diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 7bf90bf095..d0aa6fd090 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -25,6 +25,7 @@ from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.dp_attention import get_dp_global_num_tokens +from sglang.srt.layers.moe.mega_gate import run_mega_gate, should_use_mega_gate from sglang.srt.layers.moe.mega_moe_sm90 import ( is_sm90_fp8_mega_moe_available, run_sm90_mega_routed, @@ -250,6 +251,25 @@ def forward_mega_moe( return y +def _mega_moe_buffer(moe: DeepseekV2MoE): + from sglang.srt.runtime_context import get_parallel + + return _get_mega_moe_symm_buffer( + get_parallel().moe_ep_group.device_group, + num_experts=moe.experts.num_experts, + num_max_tokens_per_rank=( + envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get() + ), + num_topk=moe.config.num_experts_per_tok + moe.num_fused_shared_experts, + hidden=moe.config.hidden_size, + intermediate_hidden=moe.config.moe_intermediate_size, + num_shared_experts=( + 0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts + ), + mma_type=_mega_moe_mma_type(moe.experts), + ) + + def _run_mega_routed( moe: DeepseekV2MoE, hidden_states: torch.Tensor, @@ -259,7 +279,24 @@ def _run_mega_routed( ) -> torch.Tensor: hidden_size = moe.config.hidden_size - if num_tokens > 0: + routing = None + if should_use_mega_gate(moe, hidden_states): + import deep_gemm + + # Only the fp8xfp4 pre-dispatch can skip its top-k copy (kCopyTopk). + buf = ( + _mega_moe_buffer(moe) + if _mega_moe_mma_type(moe.experts) == "fp8xfp4" + else None + ) + with _configure_mega_moe_deep_gemm_num_sms(deep_gemm): + routing = run_mega_gate(moe, hidden_states, input_ids_global, buf) + + if routing is not None: + topk_ids = routing.topk_ids + topk_weights = routing.topk_weights + routed_scaling_factor = routing.routed_scaling_factor + elif num_tokens > 0: router_logits = moe.gate(hidden_states, forward_batch=forward_batch) topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {} if ( @@ -282,6 +319,12 @@ def _run_mega_routed( else: topk_ids = None topk_weights = None + if routing is None: + routed_scaling_factor = ( + 1.0 + if moe.routed_scaling_factor_fused_in_topk + else float(moe.routed_scaling_factor) + ) return run_mega_routed_experts( moe.experts, @@ -293,16 +336,13 @@ def _run_mega_routed( top_k=moe.config.num_experts_per_tok + moe.num_fused_shared_experts, num_tokens=num_tokens, activation_clamp=moe.experts.moe_runner_config.swiglu_limit, - routed_scaling_factor=( - 1.0 - if moe.routed_scaling_factor_fused_in_topk - else float(moe.routed_scaling_factor) - ), + routed_scaling_factor=routed_scaling_factor, num_shared_experts=( 0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts ), shared_l1_weights=moe.mega_shared_l1_weights, shared_l2_weights=moe.mega_shared_l2_weights, + routing_in_buffer=routing is not None and routing.in_buffer, ) @@ -321,6 +361,7 @@ def run_mega_routed_experts( num_shared_experts: int = 0, shared_l1_weights=None, shared_l2_weights=None, + routing_in_buffer: bool = False, ) -> torch.Tensor: # Rows are this rank's tokens; the returned rows are fully combined. import deep_gemm @@ -351,7 +392,10 @@ def run_mega_routed_experts( mma_type=mma_type, ) - if num_tokens > 0: + if routing_in_buffer: + assert num_tokens > 0 and _device_sm != 90 and mma_type == "fp8xfp4" + topk_ids_in = topk_weights_in = None + elif num_tokens > 0: topk_ids_in = topk_ids.to(torch.int32) topk_weights_in = topk_weights.to(torch.float32) else: @@ -372,6 +416,7 @@ def run_mega_routed_experts( ) if routed_scaling_factor != 1.0: + assert not routing_in_buffer topk_weights_in = topk_weights_in * routed_scaling_factor mega_kwargs = {"recipe": (1, 1, 32)} @@ -429,8 +474,8 @@ def run_mega_routed_experts( ) mega_moe_pre_dispatch( hidden_states, - topk_ids_in, - topk_weights_in, + buf.topk_idx if routing_in_buffer else topk_ids_in, + buf.topk_weights if routing_in_buffer else topk_weights_in, buf.x, buf.x_sf, buf.topk_idx, @@ -440,6 +485,7 @@ def run_mega_routed_experts( buf.shared_l1_acts_sf if shared_l1_weights is not None else None ), shared_block_m=shared_block_m, + copy_topk=not routing_in_buffer, ) # Allocate at least one row so y has a non-null CUDA data_ptr; diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 61183c5598..fdcb900f63 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -467,9 +467,11 @@ def _simulate_balanced_routing_kernel( stride_ik, stride_wm, stride_wk, + num_non_padded_ptr, K: tl.constexpr, BLOCK_K: tl.constexpr, RANDOM: tl.constexpr, + HAS_NON_PADDED: tl.constexpr = False, ): """One program per token: overwrite its top-k row with a balanced expert assignment and uniform ``1/k`` weights, in a single launch — so the @@ -505,12 +507,13 @@ def _simulate_balanced_routing_kernel( gid = (deal % num_ranks) * experts_per_rank + ( deal // num_ranks ) % experts_per_rank + w = tl.full((BLOCK_K,), inv_k, tl.float32) + if HAS_NON_PADDED: + is_pad = t >= tl.load(num_non_padded_ptr) + gid = tl.where(is_pad, -1, gid) + w = tl.where(is_pad, 0.0, w) tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask) - tl.store( - topk_weights_ptr + t * stride_wm + j * stride_wk, - tl.full((BLOCK_K,), inv_k, tl.float32), - mask=mask, - ) + tl.store(topk_weights_ptr + t * stride_wm + j * stride_wk, w, mask=mask) # Per-launch seed for the uniform (RANDOM=True) path: varies across eager calls so @@ -529,6 +532,8 @@ def _simulate_balanced_routing( token_shard_rank: int = 0, num_token_shards: int = 1, seed: Optional[int] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + weight_scale: float = 1.0, ) -> None: """Benchmark-only fused override (in place): replace ``topk_ids`` with a balanced expert assignment and ``topk_weights`` with ``1/k`` using a single @@ -589,7 +594,7 @@ def _simulate_balanced_routing( num_ranks, experts_per_rank, deal_offset, - 1.0 / k, + weight_scale / k, seed, token_shard_rank, num_token_shards, @@ -597,9 +602,11 @@ def _simulate_balanced_routing( topk_ids.stride(1), topk_weights.stride(0), topk_weights.stride(1), + num_token_non_padded if num_token_non_padded is not None else topk_ids, K=k, BLOCK_K=triton.next_power_of_2(k), RANDOM=random, + HAS_NON_PADDED=num_token_non_padded is not None, ) @@ -2401,6 +2408,8 @@ def apply_routing_simulation( num_routed_topk: int, top_k: int, layer_id: Optional[int], + num_token_non_padded: Optional[torch.Tensor] = None, + weight_scale: float = 1.0, ) -> None: simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get() # Benchmark-only: override gating with a balanced expert assignment (so @@ -2441,6 +2450,8 @@ def apply_routing_simulation( layer_id=layer_id, token_shard_rank=token_shard_rank, num_token_shards=num_token_shards, + num_token_non_padded=num_token_non_padded, + weight_scale=weight_scale, ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 60c7f4a27f..9d77d00278 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -1869,6 +1869,12 @@ class DeepseekV2MoE(nn.Module): def _routed_mxfp8_prequant_static_enabled(self) -> bool: return self._compute_routed_mxfp8_prequant_enabled()[0] + @cached_property + def mega_gate_supported(self) -> bool: + from sglang.srt.layers.moe.mega_gate import mega_gate_supported + + return mega_gate_supported(self) + def _compute_routed_mxfp8_prequant_enabled(self) -> Tuple[bool, str]: from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher diff --git a/test/registered/kernels/ops/moe/test_mega_gate.py b/test/registered/kernels/ops/moe/test_mega_gate.py new file mode 100644 index 0000000000..a1491d14a3 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_mega_gate.py @@ -0,0 +1,143 @@ +import unittest + +import torch + +from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 +from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +class TestMegaGate(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + raise unittest.SkipTest("MegaGate requires SM10x") + + @staticmethod + def _reference( + x, weight, top_k, bias, image_bias, image_mask, valid_mask, hash_ids + ): + x = torch.where(valid_mask[:, None], x, 0) + scores = torch.log1p(torch.exp(linear_bf16_fp32(x, weight))).sqrt() + if hash_ids is not None: + ids = hash_ids + else: + row_bias = ( + torch.where(image_mask[:, None], image_bias, bias) + if image_mask is not None + else bias + ) + ids = (scores + row_bias).topk(top_k, dim=-1).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(-1, keepdim=True) + 1e-20) * 1.5 + return weights.masked_fill(~valid_mask[:, None], 0), ids.masked_fill( + ~valid_mask[:, None], -1 + ) + + def _assert_routes_close(self, actual, expected): + actual_weights, actual_ids = actual + expected_weights, expected_ids = expected + actual_ids, order = actual_ids.sort(dim=-1) + actual_weights = actual_weights.gather(1, order) + expected_ids, order = expected_ids.sort(dim=-1) + expected_weights = expected_weights.gather(1, order) + torch.testing.assert_close(actual_ids, expected_ids) + torch.testing.assert_close( + actual_weights, expected_weights, rtol=1e-3, atol=1e-4 + ) + + def test_routing_and_cuda_graph_replay(self): + torch.manual_seed(42) + for num_experts, top_k in ((384, 6), (128, 6)): + for tokens in (1, 17, 512): + for routing in ("text", "image", "hash"): + with self.subTest( + num_experts=num_experts, tokens=tokens, routing=routing + ): + x = torch.randn( + (tokens, 5120), device="cuda", dtype=torch.bfloat16 + ) + weight = ( + torch.randn( + (num_experts, 5120), device="cuda", dtype=torch.bfloat16 + ) + * 0.001 + ) + bias = torch.arange(num_experts, device="cuda").float() * 0.25 + image_bias = bias.flip(0) if routing == "image" else None + image_mask = ( + torch.zeros(tokens, device="cuda", dtype=torch.bool) + if routing == "image" + else None + ) + hash_ids = ( + torch.arange(top_k, device="cuda") + .expand(tokens, -1) + .clone() + if routing == "hash" + else None + ) + valid_mask = torch.ones(tokens, device="cuda", dtype=torch.bool) + + def run(): + return bf16_mega_gate( + x, + weight, + top_k, + routed_scaling_factor=1.5, + ep_rank=0, + bias=bias, + image_bias=image_bias, + image_token_mask=image_mask, + valid_token_mask=valid_mask, + hash_topk_ids=hash_ids, + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.stream(stream): + for _ in range(3): + run() + with torch.cuda.graph(graph, stream=stream): + actual = run() + torch.cuda.current_stream().wait_stream(stream) + for valid_rows, padding in ( + (tokens, float("nan")), + (tokens // 2, float("nan")), + (0, float("inf")), + (tokens, float("nan")), + ): + x.normal_() + valid_mask.copy_( + torch.arange(tokens, device="cuda") < valid_rows + ) + x[valid_rows:] = padding + if image_mask is not None: + image_mask.logical_not_() + if hash_ids is not None: + hash_ids.copy_( + torch.arange(top_k, device="cuda") + .expand(tokens, -1) + .roll(1, dims=-1) + ) + expected = self._reference( + x, + weight, + top_k, + bias, + image_bias, + image_mask, + valid_mask, + hash_ids, + ) + graph.replay() + self._assert_routes_close(actual, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py index 29c3800b0b..a242d0d6a2 100644 --- a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py +++ b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py @@ -132,6 +132,27 @@ def test_kernel_matches_reference_on_every_shard( assert torch.equal(weights, torch.full_like(weights, 1.0 / topk)) +def test_kernel_masks_padded_rows_and_scales_weights(): + tokens, real, scale = 300, 250, 1.5 + ids = torch.zeros((tokens, TOPK), dtype=torch.int64, device="cuda") + weights = torch.ones((tokens, TOPK), dtype=torch.float32, device="cuda") + _simulate_balanced_routing( + ids, + weights, + NUM_EXPERTS, + random=False, + num_ranks=EP_SIZE, + layer_id=3, + num_token_non_padded=torch.tensor([real], device="cuda"), + weight_scale=scale, + ) + expected_ids, expected_weights = _route(tokens, layer_id=3) + assert torch.equal(ids[:real], expected_ids[:real].long()) + torch.testing.assert_close(weights[:real], expected_weights[:real] * scale) + assert torch.equal(ids[real:], torch.full_like(ids[real:], -1)) + assert torch.equal(weights[real:], torch.zeros_like(weights[real:])) + + @pytest.mark.parametrize("num_tokens", [1, 2, 4, 300]) @pytest.mark.parametrize("num_ranks", [4, 8, 16]) def test_single_shard_balances_within_one_slot(num_tokens, num_ranks): diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 0dfc57247c..b3e165b4a1 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -2,19 +2,23 @@ import sys import unittest -from contextlib import nullcontext +from contextlib import ExitStack, contextmanager, nullcontext from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock, call, patch import torch from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.environ import envs +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() -from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe +from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.layers.moe import mega_gate as mega_gate_runtime +from sglang.srt.layers.moe import mega_moe from sglang.srt.layers.moe import topk as topk_module from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module from sglang.srt.layers.moe.topk import TopK @@ -450,6 +454,219 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): self.assertEqual(out.shape, (0, 4)) moe.gate.assert_not_called() + @staticmethod + def _mega_gate_reference(x, weight, top_k, **kwargs): + if not torch.isfinite(x).all(): + raise ValueError("MegaGate rejects nonfinite GEMM inputs") + scores = torch.log1p(torch.exp(x.float() @ weight.float().T)).sqrt() + if kwargs["unmapped_topk_idx"] is not None: + ids = kwargs["unmapped_topk_idx"] + else: + bias = kwargs["bias"] + if kwargs["image_bias"] is not None: + bias = torch.where( + kwargs["image_token_mask"][:, None], kwargs["image_bias"], bias + ) + ids = (scores + bias).topk(top_k, dim=-1).indices + weights = scores.gather(1, ids) + weights = weights / (weights.sum(-1, keepdim=True) + 1e-20) + weights = weights * kwargs["routed_scaling_factor"] + if kwargs["to_physical_map"] is not None: + physical_map, count = kwargs["to_physical_map"], kwargs["logical_count"] + assert physical_map.dtype == count.dtype == torch.int32 + token = torch.arange(ids.shape[0])[:, None] + replica = (kwargs["ep_rank"] + token * 23333) % count[ids] + ids = physical_map[ids, replica] + if kwargs["mask"] is not None: + weights = weights.masked_fill(~kwargs["mask"][:, None], 0) + ids = ids.masked_fill(~kwargs["mask"][:, None], -1) + kwargs["out"][0].copy_(ids) + kwargs["out"][1].copy_(weights) + + def _mega_gate_moe(self, scale_in_topk=False, hash_routing=False): + moe = self._v41_moe(0, scale_in_topk) + moe.config.model_type = "deepseek_v41" + moe.config.hidden_size = 256 + moe.gate.weight = torch.zeros((8, 256), dtype=torch.bfloat16) + moe.gate.e_score_correction_bias = torch.tensor( + [4.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + ) + moe.gate.e_score_correction_bias_vl = torch.tensor( + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 4.0] + ) + moe.topk.topk_config.sqrtsoftplus_log1p = True + moe.topk.enable_waterfill = False + moe.mega_gate_supported = True + if hash_routing: + moe.is_hash = True + moe.gate.e_score_correction_bias = None + moe.topk = SimpleNamespace( + enable_waterfill=False, + topk=2, + num_shared_topk_slots=0, + routed_scaling_factor=2.0, + apply_routed_scaling_factor_on_output=scale_in_topk, + score_func="sqrtsoftplus", + tid2eid=torch.tensor([[6, 7]] * 100, dtype=torch.int32), + ) + return moe + + @contextmanager + def _mega_gate_env(self): + deep_gemm = ModuleType("deep_gemm") + deep_gemm.bf16_mega_gate = self._mega_gate_reference + with ExitStack() as stack: + stack.enter_context( + get_context().override_server_args( + model_path="dummy", enable_deterministic_inference=False + ) + ) + stack.enter_context(get_parallel().override(moe_ep_rank=0)) + stack.enter_context(patch.dict(sys.modules, {"deep_gemm": deep_gemm})) + stack.enter_context(patch.object(topk_module, "_is_cuda", True)) + stack.enter_context( + patch.object(topk_module, "_can_fuse_padded_region", return_value=False) + ) + yield + + def test_mega_gate_admission_falls_back_for_unsupported_routers(self): + moe = self._mega_gate_moe() + supported = mega_gate_runtime.mega_gate_supported + with ( + self._mega_gate_env(), + patch.object(mega_gate_runtime, "_device_sm", 100), + ): + self.assertTrue(supported(moe)) + self.assertTrue(supported(self._mega_gate_moe(hash_routing=True))) + with patch.object(moe.topk.topk_config, "scoring_func", "sigmoid"): + self.assertFalse(supported(moe)) + with patch.object(moe, "num_fused_shared_experts", 1): + self.assertFalse(supported(moe)) + with get_exec().deterministic.override(enable_deterministic_inference=True): + self.assertFalse(supported(moe)) + + hidden = torch.zeros((32, 256), dtype=torch.bfloat16) + with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(False): + self.assertFalse(mega_gate_runtime.should_use_mega_gate(moe, hidden)) + with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(True): + self.assertTrue(mega_gate_runtime.should_use_mega_gate(moe, hidden)) + self.assertFalse( + mega_gate_runtime.should_use_mega_gate(moe, hidden.float()) + ) + self.assertFalse( + mega_gate_runtime.should_use_mega_gate( + moe, torch.zeros((4096, 256), dtype=torch.bfloat16) + ) + ) + + def test_mega_gate_routes_vision_hash_and_padding(self): + for is_hash in (False, True): + for scale_in_topk in (False, True): + for in_buffer in (False, True): + with self.subTest( + is_hash=is_hash, + scale_in_topk=scale_in_topk, + in_buffer=in_buffer, + ): + moe = self._mega_gate_moe(scale_in_topk, is_hash) + hidden = torch.ones((4, 256), dtype=torch.bfloat16) + hidden[3:] = float("nan") + input_ids = torch.tensor([7, 99, 7, 1000]) + buf = SimpleNamespace( + topk_idx=torch.full((8, 2), 5, dtype=torch.int64), + topk_weights=torch.full((8, 2), 9.0), + ) + with ( + self._mega_gate_env(), + patch.object( + moe.topk, + "routing_inputs", + return_value=(torch.tensor([3]), None), + create=True, + ), + ): + routing = mega_gate_runtime.run_mega_gate( + moe, hidden, input_ids, buf if in_buffer else None + ) + self.assertEqual(routing.in_buffer, in_buffer) + if in_buffer: + ids, weights = buf.topk_idx[:4], buf.topk_weights[:4] + self.assertEqual(routing.routed_scaling_factor, 1.0) + self.assertTrue((buf.topk_idx[4:] == 5).all()) + scale = 2.0 + else: + ids, weights = routing.topk_ids, routing.topk_weights + self.assertEqual( + routing.routed_scaling_factor, + 1.0 if scale_in_topk else 2.0, + ) + scale = 2.0 if scale_in_topk else 1.0 + image = [6, 7] if is_hash else [7, 6] + text = [6, 7] if is_hash else [0, 1] + torch.testing.assert_close( + ids, + torch.tensor([text, image, text, [-1, -1]]), + ) + torch.testing.assert_close( + weights, + torch.tensor([[0.5 * scale] * 2] * 3 + [[0.0, 0.0]]), + ) + + def test_mega_gate_maps_eplb_in_kernel_and_records_from_buffer(self): + moe = self._mega_gate_moe() + hidden = torch.ones((4, 256), dtype=torch.bfloat16) + hidden[2:] = float("nan") + dispatch_info = ExpertLocationDispatchInfo( + ep_dispatch_algorithm="static", + partial_logical_to_rank_dispatch_physical_map=torch.tensor( + [9, 8, 7, 6, 5, 4, 3, 2] + ), + partial_logical_to_all_physical_map=None, + partial_logical_to_all_physical_map_num_valid=None, + num_physical_experts=10, + ) + recorded = [] + recorder = SimpleNamespace( + on_select_experts=lambda **kw: recorded.append(kw["topk_ids"].clone()) + ) + buf = SimpleNamespace( + topk_idx=torch.zeros((8, 2), dtype=torch.int64), + topk_weights=torch.zeros((8, 2)), + ) + with ( + self._mega_gate_env(), + patch.object( + mega_gate_runtime, + "get_global_expert_distribution_recorder", + return_value=recorder, + ), + patch.object( + moe.topk, + "routing_inputs", + return_value=(torch.tensor([2]), dispatch_info), + ), + ): + routing = mega_gate_runtime.run_mega_gate( + moe, hidden, torch.tensor([7, 99, 1000, 1000]), buf + ) + self.assertTrue(routing.in_buffer) + torch.testing.assert_close( + buf.topk_idx[:4], + torch.tensor([[9, 8], [2, 3], [-1, -1], [-1, -1]]), + ) + self.assertTrue((buf.topk_weights[2:4] == 0).all()) + self.assertEqual(len(recorded), 1) + torch.testing.assert_close(recorded[0], buf.topk_idx[:4]) + for algorithm in ("lp", "fake"): + with patch.object( + moe.topk, + "routing_inputs", + return_value=(None, SimpleNamespace(ep_dispatch_algorithm=algorithm)), + ): + self.assertIsNone( + mega_gate_runtime.run_mega_gate(moe, hidden, None, buf) + ) + @staticmethod def _v41_moe(fused_shared, scale_in_topk): with (