[GLM-5.3-Flash] Autotune MoE kernels at prefill sizes (#42693)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
620c76f25d
commit
fcbbefa82a
@@ -26,6 +26,7 @@ from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
MHCState,
|
||||
@@ -51,6 +52,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
is_shared_experts_fusion_disabled,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -1457,6 +1459,36 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
)
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
|
||||
def wants_prefill_autotune(self) -> bool:
|
||||
backend = get_moe_runner_backend()
|
||||
return (
|
||||
(backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed())
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not is_dp_attention_enabled()
|
||||
)
|
||||
|
||||
def autotune_prefill_kernels(self, num_tokens: int, *, dtype: torch.dtype) -> int:
|
||||
seen = set()
|
||||
for module in self.model.modules():
|
||||
if not isinstance(module, Glm5NextMoE):
|
||||
continue
|
||||
experts = module.experts
|
||||
key = (experts.w13_weight.shape, experts.w2_weight.shape)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
hidden_states = torch.randn(
|
||||
(num_tokens, module.gate.weight.shape[1]),
|
||||
dtype=dtype,
|
||||
device=experts.w13_weight.device,
|
||||
)
|
||||
router_logits = module.gate(hidden_states)
|
||||
experts(hidden_states, module.topk(hidden_states, router_logits))
|
||||
del hidden_states, router_logits
|
||||
if envs.SGLANG_FLASHINFER_AUTOTUNE_EXTEND.get():
|
||||
return 0
|
||||
return len(seen)
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user