minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[NPU] bugfix for W4A8MoE bias 3D dimension mismatch problem (#31707)

Browse Source
This commit is contained in:
McZyWu
2026-07-20 11:05:44 +08:00
committed by GitHub
parent 35f2d4f761
commit 1f637a65b9
1 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py
+11
View File
@@ -419,6 +419,7 @@ class NPUW4A8Int8MoEMethod(_NPUMoEMethodBase):
if scale_second is not None: if scale_second is not None:
delattr(layer, f"{weight_prefix}_weight_scale_second") delattr(layer, f"{weight_prefix}_weight_scale_second")
delattr(layer, f"{weight_prefix}_weight_offset_second") delattr(layer, f"{weight_prefix}_weight_offset_second")
self._update_bias(layer, weight_prefix)
else: else:
# With clip: simple squeeze + unsqueeze # With clip: simple squeeze + unsqueeze
processed_scale = scale.data.squeeze(-1).unsqueeze(1).contiguous() processed_scale = scale.data.squeeze(-1).unsqueeze(1).contiguous()
@@ -451,6 +452,16 @@ class NPUW4A8Int8MoEMethod(_NPUMoEMethodBase):
if weight_prefix == "w13": if weight_prefix == "w13":
self._set_dispatcher_output_dtype(layer, "int8") self._set_dispatcher_output_dtype(layer, "int8")
@staticmethod
def _update_bias(
layer: torch.nn.Module,
weight_prefix: str,
) -> None:
scale_bias_name = f"{weight_prefix}_scale_bias"
if hasattr(layer, scale_bias_name):
scale_bias = getattr(layer, scale_bias_name)
scale_bias.data = scale_bias.data.transpose(1, 2).contiguous().sum(dim=1)
def _process_scale( def _process_scale(
self, self,
weight: torch.Tensor, weight: torch.Tensor,