From 98d0e702c3e4c0428d8a1b8e0e0756d7897d9341 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Mon, 29 Jun 2026 14:19:24 -0700 Subject: [PATCH] [GLM-5] Tune the threshold of router GEMM (#29470) --- python/sglang/srt/models/deepseek_v2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 7131bcbbd..41b81fd98 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -496,9 +496,11 @@ class MoEGate(nn.Module): ): logits = F.linear(hidden_states, self.weight, None) else: + # NOTE(b8zhong): this threshold has been empirically verified + max_router_gemm_tokens = 4 if _device_sm in (100, 103) else 16 if ( _is_cuda - and hidden_states.shape[0] <= 16 + and hidden_states.shape[0] <= max_router_gemm_tokens and hidden_states.shape[1] % 1024 == 0 and (self.weight.shape[0] == 256 or self.weight.shape[0] == 384) and _device_sm >= 90