[Perf] dsv41: Run MegaGate routing in prefill (#46232)

This commit is contained in:
Liangsheng Yin
2026-10-23 20:31:50 -07:00
committed by GitHub
parent 17d92ea671
commit 3c6a6c8068
2 changed files with 15 additions and 20 deletions
+5 -5
View File
@@ -19,8 +19,6 @@ from sglang.srt.models.deepseek_common.utils import _device_sm
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
# DeepGEMM bf16_mega_gate scores one token block per launch.
_MEGA_GATE_MAX_ROWS = 2048
# Per-forward cache of the gate's row masks, installed by mega_gate_forward_scope.
_ROW_MASKS_ATTR = "_mega_gate_row_masks"
@@ -84,12 +82,14 @@ def mega_gate_supported(moe) -> bool:
def should_use_mega_gate(moe, hidden_states: torch.Tensor, forward_batch) -> bool:
if not envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() or forward_batch is None:
return False
# Prefill graph capture clears is_extend_in_batch, so gate on the mode itself.
# Prefill, decode and verify; draft extend keeps the regular router.
mode = forward_batch.forward_mode
# The routing and the Mega-mHC input go to MegaMoE's buffer, sized to its token cap.
max_rows = envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
return (
(mode.is_decode() or mode.is_target_verify())
(mode.is_decode() or mode.is_extend())
and moe.mega_gate_supported
and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS
and 0 < hidden_states.shape[0] <= max_rows
and not get_forward().sp_active
and hidden_states.dtype == torch.bfloat16
and hidden_states.is_contiguous()