[XPU] Enable fused QK-norm + RoPE for Qwen3-MoE (#40190)

This commit is contained in:
Kotthagattu Meher Sai
2026-10-04 15:45:18 +08:00
committed by GitHub
parent 7eb628612c
commit 9631f6f2d0
2 changed files with 130 additions and 6 deletions
+29 -6
View File
@@ -77,17 +77,21 @@ from sglang.srt.utils import (
is_flashinfer_available,
is_non_idle_and_non_empty,
is_npu,
is_xpu,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
_is_cuda = is_cuda()
_is_cpu = is_cpu()
_is_xpu = is_xpu()
if _is_cuda:
from sglang.kernels.ops.attention.fused_qknorm_rope import (
can_use_fused_qk_norm_rope,
fused_qk_norm_rope,
)
if _is_xpu:
from sgl_kernel import fused_inplace_qknorm_rope
@lru_cache(maxsize=1)
@@ -573,12 +577,17 @@ class Qwen3MoeAttention(nn.Module):
self.use_fused_qk_norm_rope = (
get_exec().kernel.enable_fused_qk_norm_rope
and self.compatible_with_fused_qk_norm_rope
and _is_cuda
and can_use_fused_qk_norm_rope(
self.head_dim,
self.rotary_emb.is_neox_style,
torch.bfloat16,
_yarn_factor != 1.0,
and (
_is_xpu
or (
_is_cuda
and can_use_fused_qk_norm_rope(
self.head_dim,
self.rotary_emb.is_neox_style,
torch.bfloat16,
_yarn_factor != 1.0,
)
)
)
)
self.use_fused_qk_norm_rope_cpu = (
@@ -687,6 +696,20 @@ class Qwen3MoeAttention(nn.Module):
attention_factor,
)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
elif _is_xpu:
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
fused_inplace_qknorm_rope(
q=q.view(-1, self.num_heads, self.head_dim),
k=k.view(-1, self.num_kv_heads, self.head_dim),
q_weight=self.q_norm.weight,
k_weight=self.k_norm.weight,
cos_sin_cache=self.rotary_emb.cos_sin_cache,
positions=positions.view(-1),
is_neox=self.rotary_emb.is_neox_style,
eps=self.q_norm.variance_epsilon,
head_dim=self.head_dim,
rope_dim=self.rotary_emb.rotary_dim,
)
elif _is_cpu:
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
self.rotary_emb._match_cos_sin_cache_dtype(q)