[Fix] [NPU] qwen3.5: Fix dense serving on Ascend 950 (#32745)

This commit is contained in:
Junlin Wu
2026-10-21 09:50:38 +03:00
committed by GitHub
parent 8599b7251f
commit 844d2b5dec
9 changed files with 501 additions and 29 deletions
@@ -2,6 +2,7 @@ import logging
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from sglang.srt.hardware_backend.npu.utils import NPUACLFormat, npu_format_cast
@@ -204,10 +205,15 @@ class NPUMXFP8LinearMethod(_NPULinearMethodBase):
weight = layer.weight.data
if weight.dtype == torch.float8_e4m3fn:
# Offline (ModelSlim) path: weight is already MXFP8-quantised and
# layer.weight_scale holds the uint8 block scales [out, in/32]. Only
# re-layout to [in, out] / [in//64, out, 2] strided views below.
# layer.weight_scale holds one uint8 scale per 32 input elements.
# Pair the scales for the NPU kernel, padding an odd final count
# (e.g. K=4304 -> 135 scales -> 136 -> 68 pairs).
n_dim, k_dim = layer.weight_scale.data.shape
scale = layer.weight_scale.data.reshape(n_dim, k_dim // 2, 2)
scale_data = layer.weight_scale.data
if k_dim % 2 != 0:
scale_data = F.pad(scale_data, (0, 1), mode="constant", value=0)
k_dim += 1
scale = scale_data.reshape(n_dim, k_dim // 2, 2)
layer.weight = Parameter(weight.transpose(0, 1), requires_grad=False)
layer.weight_scale_inv = Parameter(
scale.transpose(0, 1), requires_grad=False
+14 -13
View File
@@ -182,9 +182,16 @@ logger = logging.getLogger(__name__)
if _is_npu:
import torch_npu
from sgl_kernel_npu.norm.add_rmsnorm_bias import add_gemma_rms_norm
_NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE = 5120
try:
from sgl_kernel_npu.norm.gemma_rmsnorm import npu_gemma_rms_norm
except ImportError:
# sgl-kernel-npu wheels built before the target-specific Gemma provider
# landed expose only the torch_npu operator, which is exactly what that
# provider binds to on A2/A3 — so those deployments keep working
# unchanged. On Ascend 950 the operator is unregistered and the first Gemma
# forward fails loudly, same as before this indirection existed.
npu_gemma_rms_norm = torch_npu.npu_gemma_rms_norm
@lru_cache(maxsize=1)
@@ -1237,18 +1244,12 @@ class GemmaRMSNorm(BaseFusedOp):
if residual is not None:
if post_residual_addition is not None:
residual = residual + post_residual_addition
if x.shape[-1] > _NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE:
gamma = self.gemma_weight.to(x.dtype)
norm_out, _, residual = torch_npu.npu_add_rms_norm(
residual, x, gamma, self.variance_epsilon
)
else:
norm_out, residual = add_gemma_rms_norm(
x, self.weight, residual, self.variance_epsilon
)
norm_out, _, residual = torch_npu.npu_add_rms_norm(
residual, x, self.gemma_weight.to(x.dtype), self.variance_epsilon
)
return norm_out, residual
x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
x, _ = npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
return x
def forward_xpu(
@@ -1376,7 +1377,7 @@ class Gemma3RMSNorm(BaseFusedOp):
return self.forward_native(x, residual)
if residual is not None:
return self.forward_native(x, residual)
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
output, _ = npu_gemma_rms_norm(x, self.weight, self.eps)
return output
def extra_repr(self):
@@ -3,9 +3,9 @@
Loads weights pre-quantized by msmodelslim (float8_e4m3fn weights,
uint8 scales) and runs MXFP8 matmul at inference.
Following the modelslim-scheme convention (see ModelSlimW8A8Int8), this scheme
owns only the hardware-agnostic weight creation; weight post-processing and the
forward pass are delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
This scheme owns weight creation and alignment of TP shards to checkpoint
quantization blocks; weight post-processing and the forward pass are otherwise
delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
process_weights_after_loading detects the pre-quantized float8_e4m3fn weight and
takes the offline (transpose-only) branch.
"""
@@ -13,7 +13,9 @@ takes the offline (transpose-only) branch.
from typing import Dict, List, Optional
import torch
import torch.nn.functional as F
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import (
NPUMXFP8LinearMethod,
)
@@ -60,11 +62,38 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
)
layer.register_parameter("weight", weight)
# msmodelslim exports weight_scale as uint8, shape [out, in/32].
# msmodelslim exports one uint8 scale per 32 input elements. Keep the
# final partial block: Qwen3-VL has visual projections such as K=4304,
# whose checkpoint scale shape is [out, ceil(4304/32)] = [out, 135].
# NOTE: Named "weight_scale" (not "weight_scale_inv") to match the
# checkpoint key exported by msmodelslim; the kernel re-layouts it into
# weight_scale_inv during process_weights_after_loading.
scale_dim = input_size_per_partition // MXFP8_BLOCK_SIZE
scale_dim = (
input_size_per_partition + MXFP8_BLOCK_SIZE - 1
) // MXFP8_BLOCK_SIZE
scale_loader = weight_loader
if (
input_size_per_partition != input_size
and input_size_per_partition % MXFP8_BLOCK_SIZE
):
tp_rank, _ = get_group_rank_size(layer.tp_group)
start = tp_rank * input_size_per_partition
left = start % MXFP8_BLOCK_SIZE
right = -(left + input_size_per_partition) % MXFP8_BLOCK_SIZE
layer.mxfp8_tp_padding = (left, right)
scale_start = start // MXFP8_BLOCK_SIZE
scale_dim = (left + input_size_per_partition + right) // MXFP8_BLOCK_SIZE
# A shard can start inside a checkpoint block, so adjacent ranks
# may share a scale. Include that block and align both W and X to it.
def scale_loader(param, loaded_weight):
if not layer.use_presharded_weights:
loaded_weight = loaded_weight.narrow(1, scale_start, scale_dim)
assert param.shape == loaded_weight.shape, (
f"{param.shape=} {loaded_weight.shape=}"
)
param.data.copy_(loaded_weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
(output_size_per_partition, scale_dim),
@@ -72,11 +101,13 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
weight_loader=scale_loader,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module):
if padding := getattr(layer, "mxfp8_tp_padding", None):
layer.weight.data = F.pad(layer.weight.data, padding)
self.kernel.process_weights_after_loading(layer)
def apply_weights(
@@ -85,4 +116,11 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if padding := getattr(layer, "mxfp8_tp_padding", None):
if isinstance(x, tuple):
raise ValueError(
"ModelSlim MXFP8 with unaligned TP shards requires "
"unquantized inputs for block alignment."
)
x = F.pad(x, padding)
return self.kernel.apply(layer, x, bias)
@@ -300,7 +300,8 @@ class MRotaryEmbedding(RotaryEmbedding):
assert fused_set_kv_buffer_arg is None, (
"fused_set_kv_buffer_arg is not supported for npu implementation"
)
if query.shape[1] > 4096:
# The NPU call below selects ordinary RoPE, which requires 1D positions.
if positions.ndim == 2 or query.shape[1] > 4096:
return self.forward_native(positions, query, key, fused_set_kv_buffer_arg)
rotary_mode = "half" if self.is_neox_style else "interleave"
mrope_section = [0, 0, 0]
+31 -5
View File
@@ -347,6 +347,9 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
if _is_npu:
from sgl_kernel_npu.norm.split_qkv_rmsnorm_mrope import (
triton_split_qkv_rmsnorm_mrope,
)
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import (
split_qkvgate_gemma_rmsnorm_rope,
)
@@ -1541,6 +1544,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
q, k = self.rotary_emb(positions, q, k)
return q, k, v, gate
def forward_prepare_npu_prefill(self, positions, hidden_states):
rotary_emb = self.rotary_emb
if not getattr(rotary_emb, "mrope_section", None):
return self.forward_prepare_native(positions, hidden_states)
qkv, _ = self.qkv_proj(hidden_states)
if positions.ndim == 1:
positions = positions.expand(3, -1)
cos_sin = rotary_emb.cos_sin_cache[positions].to(qkv)
q, k, v, gate = triton_split_qkv_rmsnorm_mrope(
qkv=qkv,
q_weight=self.q_norm.gemma_weight,
k_weight=self.k_norm.gemma_weight,
cos_sin=cos_sin,
num_q_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
eps=self.q_norm.variance_epsilon,
mrope_section=rotary_emb.mrope_section,
is_interleaved=rotary_emb.mrope_interleaved,
rope_dim=rotary_emb.rotary_dim,
has_gate=self.attn_output_gate,
)
return q, k, v, gate if self.attn_output_gate else None
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
qkv, _ = self.qkv_proj(hidden_states)
if self.refreshes_rope_cos_sin:
@@ -1576,11 +1604,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
positions=positions,
hidden_states=hidden_states,
)
if (
not _is_npu
or forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed()
or not self.attn_output_gate
):
if _is_npu and forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed():
return self.forward_prepare_npu_prefill(positions, hidden_states)
if not _is_npu or not self.attn_output_gate:
return self.forward_prepare_native(
positions=positions,
hidden_states=hidden_states,