diff --git a/python/sglang/srt/layers/moe/mega_gate.py b/python/sglang/srt/layers/moe/mega_gate.py index 192605e8a8..3417608162 100644 --- a/python/sglang/srt/layers/moe/mega_gate.py +++ b/python/sglang/srt/layers/moe/mega_gate.py @@ -78,14 +78,16 @@ def mega_gate_supported(moe) -> bool: ) -def should_use_mega_gate(moe, hidden_states: torch.Tensor) -> bool: - forward = get_forward() +def should_use_mega_gate(moe, hidden_states: torch.Tensor, forward_batch) -> bool: + if not envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() or forward_batch is None: + return False + # Prefill graph capture clears is_extend_in_batch, so gate on the mode itself. + mode = forward_batch.forward_mode return ( - envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() + (mode.is_decode() or mode.is_target_verify()) and moe.mega_gate_supported and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS - and not forward.is_extend_in_batch - and not forward.sp_active + and not get_forward().sp_active and hidden_states.dtype == torch.bfloat16 and hidden_states.is_contiguous() ) diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index d0aa6fd090..2440f0fed4 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -280,7 +280,7 @@ def _run_mega_routed( hidden_size = moe.config.hidden_size routing = None - if should_use_mega_gate(moe, hidden_states): + if should_use_mega_gate(moe, hidden_states, forward_batch): import deep_gemm # Only the fp8xfp4 pre-dispatch can skip its top-k copy (kCopyTopk). diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 09a731c6e9..7364d8ffdd 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -24,11 +24,13 @@ from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_modu from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import draft_model_build_scope from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.runtime_context import ( get_context, get_exec, get_flags, + get_forward, get_model, get_parallel, ) @@ -547,18 +549,33 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): self.assertFalse(supported(moe)) hidden = torch.zeros((32, 256), dtype=torch.bfloat16) + use = mega_gate_runtime.should_use_mega_gate + decode = SimpleNamespace(forward_mode=ForwardMode.DECODE) with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(False): - self.assertFalse(mega_gate_runtime.should_use_mega_gate(moe, hidden)) + self.assertFalse(use(moe, hidden, decode)) with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(True): - self.assertTrue(mega_gate_runtime.should_use_mega_gate(moe, hidden)) - self.assertFalse( - mega_gate_runtime.should_use_mega_gate(moe, hidden.float()) - ) - self.assertFalse( - mega_gate_runtime.should_use_mega_gate( - moe, torch.zeros((4096, 256), dtype=torch.bfloat16) + self.assertTrue(use(moe, hidden, decode)) + self.assertTrue( + use( + moe, + hidden, + SimpleNamespace(forward_mode=ForwardMode.TARGET_VERIFY), ) ) + self.assertFalse(use(moe, hidden.float(), decode)) + self.assertFalse( + use(moe, torch.zeros((4096, 256), dtype=torch.bfloat16), decode) + ) + # Prefill graph capture runs with is_extend_in_batch cleared. + with get_forward().scoped(is_extend_in_batch=False): + self.assertFalse( + use( + moe, + hidden, + SimpleNamespace(forward_mode=ForwardMode.EXTEND), + ) + ) + self.assertFalse(use(moe, hidden, None)) def test_mega_gate_routes_vision_hash_and_padding(self): for is_hash in (False, True):