[Fix] [NPU] qwen3.5: Fix dense serving on Ascend 950 (#32745)
This commit is contained in:
@@ -2,6 +2,7 @@ import logging
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from sglang.srt.hardware_backend.npu.utils import NPUACLFormat, npu_format_cast
|
||||
@@ -204,10 +205,15 @@ class NPUMXFP8LinearMethod(_NPULinearMethodBase):
|
||||
weight = layer.weight.data
|
||||
if weight.dtype == torch.float8_e4m3fn:
|
||||
# Offline (ModelSlim) path: weight is already MXFP8-quantised and
|
||||
# layer.weight_scale holds the uint8 block scales [out, in/32]. Only
|
||||
# re-layout to [in, out] / [in//64, out, 2] strided views below.
|
||||
# layer.weight_scale holds one uint8 scale per 32 input elements.
|
||||
# Pair the scales for the NPU kernel, padding an odd final count
|
||||
# (e.g. K=4304 -> 135 scales -> 136 -> 68 pairs).
|
||||
n_dim, k_dim = layer.weight_scale.data.shape
|
||||
scale = layer.weight_scale.data.reshape(n_dim, k_dim // 2, 2)
|
||||
scale_data = layer.weight_scale.data
|
||||
if k_dim % 2 != 0:
|
||||
scale_data = F.pad(scale_data, (0, 1), mode="constant", value=0)
|
||||
k_dim += 1
|
||||
scale = scale_data.reshape(n_dim, k_dim // 2, 2)
|
||||
layer.weight = Parameter(weight.transpose(0, 1), requires_grad=False)
|
||||
layer.weight_scale_inv = Parameter(
|
||||
scale.transpose(0, 1), requires_grad=False
|
||||
|
||||
@@ -182,9 +182,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if _is_npu:
|
||||
import torch_npu
|
||||
from sgl_kernel_npu.norm.add_rmsnorm_bias import add_gemma_rms_norm
|
||||
|
||||
_NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE = 5120
|
||||
try:
|
||||
from sgl_kernel_npu.norm.gemma_rmsnorm import npu_gemma_rms_norm
|
||||
except ImportError:
|
||||
# sgl-kernel-npu wheels built before the target-specific Gemma provider
|
||||
# landed expose only the torch_npu operator, which is exactly what that
|
||||
# provider binds to on A2/A3 — so those deployments keep working
|
||||
# unchanged. On Ascend 950 the operator is unregistered and the first Gemma
|
||||
# forward fails loudly, same as before this indirection existed.
|
||||
npu_gemma_rms_norm = torch_npu.npu_gemma_rms_norm
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -1237,18 +1244,12 @@ class GemmaRMSNorm(BaseFusedOp):
|
||||
if residual is not None:
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
if x.shape[-1] > _NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE:
|
||||
gamma = self.gemma_weight.to(x.dtype)
|
||||
norm_out, _, residual = torch_npu.npu_add_rms_norm(
|
||||
residual, x, gamma, self.variance_epsilon
|
||||
)
|
||||
else:
|
||||
norm_out, residual = add_gemma_rms_norm(
|
||||
x, self.weight, residual, self.variance_epsilon
|
||||
)
|
||||
norm_out, _, residual = torch_npu.npu_add_rms_norm(
|
||||
residual, x, self.gemma_weight.to(x.dtype), self.variance_epsilon
|
||||
)
|
||||
return norm_out, residual
|
||||
|
||||
x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
|
||||
x, _ = npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
|
||||
return x
|
||||
|
||||
def forward_xpu(
|
||||
@@ -1376,7 +1377,7 @@ class Gemma3RMSNorm(BaseFusedOp):
|
||||
return self.forward_native(x, residual)
|
||||
if residual is not None:
|
||||
return self.forward_native(x, residual)
|
||||
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
|
||||
output, _ = npu_gemma_rms_norm(x, self.weight, self.eps)
|
||||
return output
|
||||
|
||||
def extra_repr(self):
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
Loads weights pre-quantized by msmodelslim (float8_e4m3fn weights,
|
||||
uint8 scales) and runs MXFP8 matmul at inference.
|
||||
|
||||
Following the modelslim-scheme convention (see ModelSlimW8A8Int8), this scheme
|
||||
owns only the hardware-agnostic weight creation; weight post-processing and the
|
||||
forward pass are delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
|
||||
This scheme owns weight creation and alignment of TP shards to checkpoint
|
||||
quantization blocks; weight post-processing and the forward pass are otherwise
|
||||
delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
|
||||
process_weights_after_loading detects the pre-quantized float8_e4m3fn weight and
|
||||
takes the offline (transpose-only) branch.
|
||||
"""
|
||||
@@ -13,7 +13,9 @@ takes the offline (transpose-only) branch.
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import (
|
||||
NPUMXFP8LinearMethod,
|
||||
)
|
||||
@@ -60,11 +62,38 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# msmodelslim exports weight_scale as uint8, shape [out, in/32].
|
||||
# msmodelslim exports one uint8 scale per 32 input elements. Keep the
|
||||
# final partial block: Qwen3-VL has visual projections such as K=4304,
|
||||
# whose checkpoint scale shape is [out, ceil(4304/32)] = [out, 135].
|
||||
# NOTE: Named "weight_scale" (not "weight_scale_inv") to match the
|
||||
# checkpoint key exported by msmodelslim; the kernel re-layouts it into
|
||||
# weight_scale_inv during process_weights_after_loading.
|
||||
scale_dim = input_size_per_partition // MXFP8_BLOCK_SIZE
|
||||
scale_dim = (
|
||||
input_size_per_partition + MXFP8_BLOCK_SIZE - 1
|
||||
) // MXFP8_BLOCK_SIZE
|
||||
scale_loader = weight_loader
|
||||
if (
|
||||
input_size_per_partition != input_size
|
||||
and input_size_per_partition % MXFP8_BLOCK_SIZE
|
||||
):
|
||||
tp_rank, _ = get_group_rank_size(layer.tp_group)
|
||||
start = tp_rank * input_size_per_partition
|
||||
left = start % MXFP8_BLOCK_SIZE
|
||||
right = -(left + input_size_per_partition) % MXFP8_BLOCK_SIZE
|
||||
layer.mxfp8_tp_padding = (left, right)
|
||||
scale_start = start // MXFP8_BLOCK_SIZE
|
||||
scale_dim = (left + input_size_per_partition + right) // MXFP8_BLOCK_SIZE
|
||||
|
||||
# A shard can start inside a checkpoint block, so adjacent ranks
|
||||
# may share a scale. Include that block and align both W and X to it.
|
||||
def scale_loader(param, loaded_weight):
|
||||
if not layer.use_presharded_weights:
|
||||
loaded_weight = loaded_weight.narrow(1, scale_start, scale_dim)
|
||||
assert param.shape == loaded_weight.shape, (
|
||||
f"{param.shape=} {loaded_weight.shape=}"
|
||||
)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
(output_size_per_partition, scale_dim),
|
||||
@@ -72,11 +101,13 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
weight_loader=scale_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
if padding := getattr(layer, "mxfp8_tp_padding", None):
|
||||
layer.weight.data = F.pad(layer.weight.data, padding)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
@@ -85,4 +116,11 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if padding := getattr(layer, "mxfp8_tp_padding", None):
|
||||
if isinstance(x, tuple):
|
||||
raise ValueError(
|
||||
"ModelSlim MXFP8 with unaligned TP shards requires "
|
||||
"unquantized inputs for block alignment."
|
||||
)
|
||||
x = F.pad(x, padding)
|
||||
return self.kernel.apply(layer, x, bias)
|
||||
|
||||
@@ -300,7 +300,8 @@ class MRotaryEmbedding(RotaryEmbedding):
|
||||
assert fused_set_kv_buffer_arg is None, (
|
||||
"fused_set_kv_buffer_arg is not supported for npu implementation"
|
||||
)
|
||||
if query.shape[1] > 4096:
|
||||
# The NPU call below selects ordinary RoPE, which requires 1D positions.
|
||||
if positions.ndim == 2 or query.shape[1] > 4096:
|
||||
return self.forward_native(positions, query, key, fused_set_kv_buffer_arg)
|
||||
rotary_mode = "half" if self.is_neox_style else "interleave"
|
||||
mrope_section = [0, 0, 0]
|
||||
|
||||
@@ -347,6 +347,9 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.norm.split_qkv_rmsnorm_mrope import (
|
||||
triton_split_qkv_rmsnorm_mrope,
|
||||
)
|
||||
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import (
|
||||
split_qkvgate_gemma_rmsnorm_rope,
|
||||
)
|
||||
@@ -1541,6 +1544,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
return q, k, v, gate
|
||||
|
||||
def forward_prepare_npu_prefill(self, positions, hidden_states):
|
||||
rotary_emb = self.rotary_emb
|
||||
if not getattr(rotary_emb, "mrope_section", None):
|
||||
return self.forward_prepare_native(positions, hidden_states)
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if positions.ndim == 1:
|
||||
positions = positions.expand(3, -1)
|
||||
cos_sin = rotary_emb.cos_sin_cache[positions].to(qkv)
|
||||
q, k, v, gate = triton_split_qkv_rmsnorm_mrope(
|
||||
qkv=qkv,
|
||||
q_weight=self.q_norm.gemma_weight,
|
||||
k_weight=self.k_norm.gemma_weight,
|
||||
cos_sin=cos_sin,
|
||||
num_q_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
eps=self.q_norm.variance_epsilon,
|
||||
mrope_section=rotary_emb.mrope_section,
|
||||
is_interleaved=rotary_emb.mrope_interleaved,
|
||||
rope_dim=rotary_emb.rotary_dim,
|
||||
has_gate=self.attn_output_gate,
|
||||
)
|
||||
return q, k, v, gate if self.attn_output_gate else None
|
||||
|
||||
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if self.refreshes_rope_cos_sin:
|
||||
@@ -1576,11 +1604,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
if (
|
||||
not _is_npu
|
||||
or forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed()
|
||||
or not self.attn_output_gate
|
||||
):
|
||||
if _is_npu and forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed():
|
||||
return self.forward_prepare_npu_prefill(positions, hidden_states)
|
||||
if not _is_npu or not self.attn_output_gate:
|
||||
return self.forward_prepare_native(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user