[npu] decoding procedure optimization on qwen3.5/3.6 (#35958)
This commit is contained in:
@@ -19,12 +19,11 @@ if not is_cpu():
|
||||
|
||||
if is_npu():
|
||||
from sgl_kernel_npu.fla.chunk import chunk_gated_delta_rule_npu
|
||||
from sgl_kernel_npu.fla.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update_npu,
|
||||
from sgl_kernel_npu.fla.fused_sigmoid_gating_recurrent_decode_optimized import (
|
||||
fused_sigmoid_gating_delta_rule_update_decode_npu as fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
|
||||
chunk_gated_delta_rule = chunk_gated_delta_rule_npu
|
||||
fused_sigmoid_gating_delta_rule_update = fused_sigmoid_gating_delta_rule_update_npu
|
||||
elif is_cpu():
|
||||
from sgl_kernel.mamba import chunk_gated_delta_rule_cpu
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.distributed import (
|
||||
moe_tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -108,14 +109,6 @@ from sglang.srt.utils import (
|
||||
make_layers,
|
||||
use_intel_amx_backend,
|
||||
)
|
||||
|
||||
if is_npu():
|
||||
from sglang.srt.hardware_backend.npu.cmo import (
|
||||
shared_expert_on_independent_stream,
|
||||
wait_share_stream,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
|
||||
@@ -126,6 +119,7 @@ _is_cuda = is_cuda()
|
||||
_is_cpu = is_cpu()
|
||||
_is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
|
||||
|
||||
@@ -539,6 +533,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
|
||||
gate = self.shared_expert_gate(hidden_states)
|
||||
shared_output = sigmoid_gate_mul_broadcast(shared_output, gate)
|
||||
elif _is_npu:
|
||||
from sgl_kernel_npu.activation.fused_sigmoid_mul import (
|
||||
fused_sigmoid_mul_broadcast,
|
||||
)
|
||||
|
||||
gate = self.shared_expert_gate(hidden_states)
|
||||
shared_output = fused_sigmoid_mul_broadcast(shared_output, gate)
|
||||
else:
|
||||
shared_output = (
|
||||
F.sigmoid(self.shared_expert_gate(hidden_states))
|
||||
@@ -585,6 +586,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
if enable_dual_stream:
|
||||
from sglang.srt.hardware_backend.npu.cmo import (
|
||||
shared_expert_on_independent_stream,
|
||||
wait_share_stream,
|
||||
)
|
||||
|
||||
shared_output = shared_expert_on_independent_stream(
|
||||
hidden_states.clone(), self._forward_shared_experts
|
||||
)
|
||||
|
||||
@@ -245,6 +245,15 @@ if _is_cpu:
|
||||
torch.ops.sgl_kernel.fused_qkvzba_split_reshape_cat_contiguous_cpu
|
||||
)
|
||||
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.activation.fused_sigmoid_mul import (
|
||||
fused_sigmoid_mul as npu_fused_sigmoid_mul,
|
||||
)
|
||||
|
||||
# NPU uses the Ascend-tuned implementation; other backends keep the
|
||||
# original Triton kernel.
|
||||
from sgl_kernel_npu.fla.utils import fused_qkvzba_split_reshape_cat_contiguous
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _enable_qwen35_fused_ar_quant() -> bool:
|
||||
@@ -984,7 +993,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
z = None
|
||||
b = projected_states_ba
|
||||
a = projected_states_ba
|
||||
elif use_fused_contiguous_unpack and not _is_npu:
|
||||
elif use_fused_contiguous_unpack:
|
||||
if _is_cpu:
|
||||
num_k_heads_tp = self.num_k_heads // self.attn_tp_size
|
||||
num_v_heads_tp = self.num_v_heads // self.attn_tp_size
|
||||
@@ -1579,7 +1588,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
attn_output = fused_sigmoid_mul(attn_output, gate, inplace=True)
|
||||
else:
|
||||
gate_val = gate.reshape(gate.shape[0], -1) if gate.ndim == 3 else gate
|
||||
attn_output.mul_(torch.sigmoid(gate_val))
|
||||
attn_output = npu_fused_sigmoid_mul(attn_output, gate_val.contiguous())
|
||||
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user