diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index f9f919100b..8bdf599b69 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -26,6 +26,7 @@ from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.conv import Conv2dLayer +from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import ( PLAIN_RESIDUAL_OPS, MHCState, @@ -51,6 +52,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.utils import ( get_moe_a2a_backend, + get_moe_runner_backend, is_shared_experts_fusion_disabled, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -1457,6 +1459,36 @@ class Glm5NextForConditionalGeneration(nn.Module): ) log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") + def wants_prefill_autotune(self) -> bool: + backend = get_moe_runner_backend() + return ( + (backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed()) + and get_moe_a2a_backend().is_none() + and not is_dp_attention_enabled() + ) + + def autotune_prefill_kernels(self, num_tokens: int, *, dtype: torch.dtype) -> int: + seen = set() + for module in self.model.modules(): + if not isinstance(module, Glm5NextMoE): + continue + experts = module.experts + key = (experts.w13_weight.shape, experts.w2_weight.shape) + if key in seen: + continue + seen.add(key) + hidden_states = torch.randn( + (num_tokens, module.gate.weight.shape[1]), + dtype=dtype, + device=experts.w13_weight.device, + ) + router_logits = module.gate(hidden_states) + experts(hidden_states, module.topk(hidden_states, router_logits)) + del hidden_states, router_logits + if envs.SGLANG_FLASHINFER_AUTOTUNE_EXTEND.get(): + return 0 + return len(seen) + def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): if not self.pp_group.is_last_rank: return