From 3c6a6c8068658b88e40bcaa5a0bf48b9a94ebdba Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 23 Oct 2026 20:31:50 -0700 Subject: [PATCH] [Perf] dsv41: Run MegaGate routing in prefill (#46232) --- python/sglang/srt/layers/moe/mega_gate.py | 10 ++++---- .../layers/moe/test_mega_moe_deepgemm_api.py | 25 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/python/sglang/srt/layers/moe/mega_gate.py b/python/sglang/srt/layers/moe/mega_gate.py index 0376a93b35..3eb31f941f 100644 --- a/python/sglang/srt/layers/moe/mega_gate.py +++ b/python/sglang/srt/layers/moe/mega_gate.py @@ -19,8 +19,6 @@ from sglang.srt.models.deepseek_common.utils import _device_sm from sglang.srt.runtime_context import get_exec, get_forward, get_parallel from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer -# DeepGEMM bf16_mega_gate scores one token block per launch. -_MEGA_GATE_MAX_ROWS = 2048 # Per-forward cache of the gate's row masks, installed by mega_gate_forward_scope. _ROW_MASKS_ATTR = "_mega_gate_row_masks" @@ -84,12 +82,14 @@ def mega_gate_supported(moe) -> bool: def should_use_mega_gate(moe, hidden_states: torch.Tensor, forward_batch) -> bool: if not envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get() or forward_batch is None: return False - # Prefill graph capture clears is_extend_in_batch, so gate on the mode itself. + # Prefill, decode and verify; draft extend keeps the regular router. mode = forward_batch.forward_mode + # The routing and the Mega-mHC input go to MegaMoE's buffer, sized to its token cap. + max_rows = envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get() return ( - (mode.is_decode() or mode.is_target_verify()) + (mode.is_decode() or mode.is_extend()) and moe.mega_gate_supported - and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS + and 0 < hidden_states.shape[0] <= max_rows and not get_forward().sp_active and hidden_states.dtype == torch.bfloat16 and hidden_states.is_contiguous() diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 7364d8ffdd..4b5be67b73 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -30,7 +30,6 @@ from sglang.srt.runtime_context import ( get_context, get_exec, get_flags, - get_forward, get_model, get_parallel, ) @@ -555,26 +554,22 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): self.assertFalse(use(moe, hidden, decode)) with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(True): self.assertTrue(use(moe, hidden, decode)) - self.assertTrue( + for mode in (ForwardMode.TARGET_VERIFY, ForwardMode.EXTEND): + self.assertTrue( + use(moe, hidden, SimpleNamespace(forward_mode=mode)) + ) + self.assertFalse( use( moe, hidden, - SimpleNamespace(forward_mode=ForwardMode.TARGET_VERIFY), + SimpleNamespace(forward_mode=ForwardMode.DRAFT_EXTEND_V2), ) ) self.assertFalse(use(moe, hidden.float(), decode)) - self.assertFalse( - use(moe, torch.zeros((4096, 256), dtype=torch.bfloat16), decode) - ) - # Prefill graph capture runs with is_extend_in_batch cleared. - with get_forward().scoped(is_extend_in_batch=False): - self.assertFalse( - use( - moe, - hidden, - SimpleNamespace(forward_mode=ForwardMode.EXTEND), - ) - ) + with envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.override( + 16 + ): + self.assertFalse(use(moe, hidden, decode)) self.assertFalse(use(moe, hidden, None)) def test_mega_gate_routes_vision_hash_and_padding(self):