[XPU] Enable fused QK-norm + RoPE for Qwen3-MoE (#40190)
This commit is contained in:
@@ -77,17 +77,21 @@ from sglang.srt.utils import (
|
||||
is_flashinfer_available,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_cpu = is_cpu()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.attention.fused_qknorm_rope import (
|
||||
can_use_fused_qk_norm_rope,
|
||||
fused_qk_norm_rope,
|
||||
)
|
||||
if _is_xpu:
|
||||
from sgl_kernel import fused_inplace_qknorm_rope
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -573,12 +577,17 @@ class Qwen3MoeAttention(nn.Module):
|
||||
self.use_fused_qk_norm_rope = (
|
||||
get_exec().kernel.enable_fused_qk_norm_rope
|
||||
and self.compatible_with_fused_qk_norm_rope
|
||||
and _is_cuda
|
||||
and can_use_fused_qk_norm_rope(
|
||||
self.head_dim,
|
||||
self.rotary_emb.is_neox_style,
|
||||
torch.bfloat16,
|
||||
_yarn_factor != 1.0,
|
||||
and (
|
||||
_is_xpu
|
||||
or (
|
||||
_is_cuda
|
||||
and can_use_fused_qk_norm_rope(
|
||||
self.head_dim,
|
||||
self.rotary_emb.is_neox_style,
|
||||
torch.bfloat16,
|
||||
_yarn_factor != 1.0,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
self.use_fused_qk_norm_rope_cpu = (
|
||||
@@ -687,6 +696,20 @@ class Qwen3MoeAttention(nn.Module):
|
||||
attention_factor,
|
||||
)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
elif _is_xpu:
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
fused_inplace_qknorm_rope(
|
||||
q=q.view(-1, self.num_heads, self.head_dim),
|
||||
k=k.view(-1, self.num_kv_heads, self.head_dim),
|
||||
q_weight=self.q_norm.weight,
|
||||
k_weight=self.k_norm.weight,
|
||||
cos_sin_cache=self.rotary_emb.cos_sin_cache,
|
||||
positions=positions.view(-1),
|
||||
is_neox=self.rotary_emb.is_neox_style,
|
||||
eps=self.q_norm.variance_epsilon,
|
||||
head_dim=self.head_dim,
|
||||
rope_dim=self.rotary_emb.rotary_dim,
|
||||
)
|
||||
elif _is_cpu:
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
self.rotary_emb._match_cos_sin_cache_dtype(q)
|
||||
|
||||
Reference in New Issue
Block a user