[GLM-5.3-Flash] Autotune MoE kernels at prefill sizes (#42693)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-05 21:04:54 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent 620c76f25d
commit fcbbefa82a
+32
View File
@@ -26,6 +26,7 @@ from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL_OPS,
MHCState,
@@ -51,6 +52,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
get_moe_runner_backend,
is_shared_experts_fusion_disabled,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -1457,6 +1459,36 @@ class Glm5NextForConditionalGeneration(nn.Module):
)
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
def wants_prefill_autotune(self) -> bool:
backend = get_moe_runner_backend()
return (
(backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed())
and get_moe_a2a_backend().is_none()
and not is_dp_attention_enabled()
)
def autotune_prefill_kernels(self, num_tokens: int, *, dtype: torch.dtype) -> int:
seen = set()
for module in self.model.modules():
if not isinstance(module, Glm5NextMoE):
continue
experts = module.experts
key = (experts.w13_weight.shape, experts.w2_weight.shape)
if key in seen:
continue
seen.add(key)
hidden_states = torch.randn(
(num_tokens, module.gate.weight.shape[1]),
dtype=dtype,
device=experts.w13_weight.device,
)
router_logits = module.gate(hidden_states)
experts(hidden_states, module.topk(hidden_states, router_logits))
del hidden_states, router_logits
if envs.SGLANG_FLASHINFER_AUTOTUNE_EXTEND.get():
return 0
return len(seen)
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
if not self.pp_group.is_last_rank:
return