[Perf] dsv41: Run MegaGate routing in prefill (#46232)
This commit is contained in:
@@ -19,8 +19,6 @@ from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
|
||||
# DeepGEMM bf16_mega_gate scores one token block per launch.
|
||||
_MEGA_GATE_MAX_ROWS = 2048
|
||||
# Per-forward cache of the gate's row masks, installed by mega_gate_forward_scope.
|
||||
_ROW_MASKS_ATTR = "_mega_gate_row_masks"
|
||||
|
||||
@@ -84,12 +82,14 @@ def mega_gate_supported(moe) -> bool:
|
||||
def should_use_mega_gate(moe, hidden_states: torch.Tensor, forward_batch) -> bool:
|
||||
if not envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() or forward_batch is None:
|
||||
return False
|
||||
# Prefill graph capture clears is_extend_in_batch, so gate on the mode itself.
|
||||
# Prefill, decode and verify; draft extend keeps the regular router.
|
||||
mode = forward_batch.forward_mode
|
||||
# The routing and the Mega-mHC input go to MegaMoE's buffer, sized to its token cap.
|
||||
max_rows = envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
|
||||
return (
|
||||
(mode.is_decode() or mode.is_target_verify())
|
||||
(mode.is_decode() or mode.is_extend())
|
||||
and moe.mega_gate_supported
|
||||
and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS
|
||||
and 0 < hidden_states.shape[0] <= max_rows
|
||||
and not get_forward().sp_active
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and hidden_states.is_contiguous()
|
||||
|
||||
Reference in New Issue
Block a user