[npu] decoding procedure optimization on qwen3.5/3.6 (#35958)

This commit is contained in:
MatsueYu
2026-09-23 15:23:51 +08:00
committed by GitHub
parent 0b0f947265
commit d4dcce12d4
3 changed files with 27 additions and 13 deletions
@@ -19,12 +19,11 @@ if not is_cpu():
if is_npu():
from sgl_kernel_npu.fla.chunk import chunk_gated_delta_rule_npu
from sgl_kernel_npu.fla.fused_sigmoid_gating_recurrent import (
fused_sigmoid_gating_delta_rule_update_npu,
from sgl_kernel_npu.fla.fused_sigmoid_gating_recurrent_decode_optimized import (
fused_sigmoid_gating_delta_rule_update_decode_npu as fused_sigmoid_gating_delta_rule_update,
)
chunk_gated_delta_rule = chunk_gated_delta_rule_npu
fused_sigmoid_gating_delta_rule_update = fused_sigmoid_gating_delta_rule_update_npu
elif is_cpu():
from sgl_kernel.mamba import chunk_gated_delta_rule_cpu
+14 -8
View File
@@ -38,6 +38,7 @@ from sglang.srt.distributed import (
moe_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -108,14 +109,6 @@ from sglang.srt.utils import (
make_layers,
use_intel_amx_backend,
)
if is_npu():
from sglang.srt.hardware_backend.npu.cmo import (
shared_expert_on_independent_stream,
wait_share_stream,
)
from sglang.srt.environ import envs
from sglang.srt.utils.hf_transformers_utils import get_rope_config
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
@@ -126,6 +119,7 @@ _is_cuda = is_cuda()
_is_cpu = is_cpu()
_is_cpu_amx_available = cpu_has_amx_support()
_is_hip = is_hip()
_is_npu = is_npu()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
@@ -539,6 +533,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
gate = self.shared_expert_gate(hidden_states)
shared_output = sigmoid_gate_mul_broadcast(shared_output, gate)
elif _is_npu:
from sgl_kernel_npu.activation.fused_sigmoid_mul import (
fused_sigmoid_mul_broadcast,
)
gate = self.shared_expert_gate(hidden_states)
shared_output = fused_sigmoid_mul_broadcast(shared_output, gate)
else:
shared_output = (
F.sigmoid(self.shared_expert_gate(hidden_states))
@@ -585,6 +586,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.gate(hidden_states)
if enable_dual_stream:
from sglang.srt.hardware_backend.npu.cmo import (
shared_expert_on_independent_stream,
wait_share_stream,
)
shared_output = shared_expert_on_independent_stream(
hidden_states.clone(), self._forward_shared_experts
)
+11 -2
View File
@@ -245,6 +245,15 @@ if _is_cpu:
torch.ops.sgl_kernel.fused_qkvzba_split_reshape_cat_contiguous_cpu
)
if _is_npu:
from sgl_kernel_npu.activation.fused_sigmoid_mul import (
fused_sigmoid_mul as npu_fused_sigmoid_mul,
)
# NPU uses the Ascend-tuned implementation; other backends keep the
# original Triton kernel.
from sgl_kernel_npu.fla.utils import fused_qkvzba_split_reshape_cat_contiguous
@lru_cache(maxsize=1)
def _enable_qwen35_fused_ar_quant() -> bool:
@@ -984,7 +993,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
z = None
b = projected_states_ba
a = projected_states_ba
elif use_fused_contiguous_unpack and not _is_npu:
elif use_fused_contiguous_unpack:
if _is_cpu:
num_k_heads_tp = self.num_k_heads // self.attn_tp_size
num_v_heads_tp = self.num_v_heads // self.attn_tp_size
@@ -1579,7 +1588,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
attn_output = fused_sigmoid_mul(attn_output, gate, inplace=True)
else:
gate_val = gate.reshape(gate.shape[0], -1) if gate.ndim == 3 else gate
attn_output.mul_(torch.sigmoid(gate_val))
attn_output = npu_fused_sigmoid_mul(attn_output, gate_val.contiguous())
output, _ = self.o_proj(attn_output)
return output