[Fix] [NPU] qwen3.5: Fix dense serving on Ascend 950 (#32745)

This commit is contained in:
Junlin Wu
2026-10-21 09:50:38 +03:00
committed by GitHub
parent 8599b7251f
commit 844d2b5dec
9 changed files with 501 additions and 29 deletions
@@ -245,6 +245,18 @@ For installation of Triton on Ascend nightly builds or from sources, follow [ins
We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md).
<Note>
Gemma-family models (including Qwen3.5) call `sgl_kernel_npu.norm.gemma_rmsnorm`,
whose implementation is chosen when the wheel is built: the native `torch_npu`
operator for Ascend 910B/910C, and ACLNN RMSNorm for Ascend 950, which does not
register that operator. **Ascend 950 users must install the `950` release
package** (`950` appears in the asset name), or build from source with
`bash build.sh -a kernels Ascend950PR_9599`, the target that package is built
with. On Ascend 950, a `910b` or `a3` package fails on the first Gemma forward,
and so does a package from a release that predates this module; check with
`python -c "import sgl_kernel_npu.norm.gemma_rmsnorm"`.
</Note>
#### DeepEP-compatible Library
We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md).
@@ -2,6 +2,7 @@ import logging
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from sglang.srt.hardware_backend.npu.utils import NPUACLFormat, npu_format_cast
@@ -204,10 +205,15 @@ class NPUMXFP8LinearMethod(_NPULinearMethodBase):
weight = layer.weight.data
if weight.dtype == torch.float8_e4m3fn:
# Offline (ModelSlim) path: weight is already MXFP8-quantised and
# layer.weight_scale holds the uint8 block scales [out, in/32]. Only
# re-layout to [in, out] / [in//64, out, 2] strided views below.
# layer.weight_scale holds one uint8 scale per 32 input elements.
# Pair the scales for the NPU kernel, padding an odd final count
# (e.g. K=4304 -> 135 scales -> 136 -> 68 pairs).
n_dim, k_dim = layer.weight_scale.data.shape
scale = layer.weight_scale.data.reshape(n_dim, k_dim // 2, 2)
scale_data = layer.weight_scale.data
if k_dim % 2 != 0:
scale_data = F.pad(scale_data, (0, 1), mode="constant", value=0)
k_dim += 1
scale = scale_data.reshape(n_dim, k_dim // 2, 2)
layer.weight = Parameter(weight.transpose(0, 1), requires_grad=False)
layer.weight_scale_inv = Parameter(
scale.transpose(0, 1), requires_grad=False
+14 -13
View File
@@ -182,9 +182,16 @@ logger = logging.getLogger(__name__)
if _is_npu:
import torch_npu
from sgl_kernel_npu.norm.add_rmsnorm_bias import add_gemma_rms_norm
_NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE = 5120
try:
from sgl_kernel_npu.norm.gemma_rmsnorm import npu_gemma_rms_norm
except ImportError:
# sgl-kernel-npu wheels built before the target-specific Gemma provider
# landed expose only the torch_npu operator, which is exactly what that
# provider binds to on A2/A3 — so those deployments keep working
# unchanged. On Ascend 950 the operator is unregistered and the first Gemma
# forward fails loudly, same as before this indirection existed.
npu_gemma_rms_norm = torch_npu.npu_gemma_rms_norm
@lru_cache(maxsize=1)
@@ -1237,18 +1244,12 @@ class GemmaRMSNorm(BaseFusedOp):
if residual is not None:
if post_residual_addition is not None:
residual = residual + post_residual_addition
if x.shape[-1] > _NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE:
gamma = self.gemma_weight.to(x.dtype)
norm_out, _, residual = torch_npu.npu_add_rms_norm(
residual, x, gamma, self.variance_epsilon
)
else:
norm_out, residual = add_gemma_rms_norm(
x, self.weight, residual, self.variance_epsilon
)
norm_out, _, residual = torch_npu.npu_add_rms_norm(
residual, x, self.gemma_weight.to(x.dtype), self.variance_epsilon
)
return norm_out, residual
x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
x, _ = npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
return x
def forward_xpu(
@@ -1376,7 +1377,7 @@ class Gemma3RMSNorm(BaseFusedOp):
return self.forward_native(x, residual)
if residual is not None:
return self.forward_native(x, residual)
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
output, _ = npu_gemma_rms_norm(x, self.weight, self.eps)
return output
def extra_repr(self):
@@ -3,9 +3,9 @@
Loads weights pre-quantized by msmodelslim (float8_e4m3fn weights,
uint8 scales) and runs MXFP8 matmul at inference.
Following the modelslim-scheme convention (see ModelSlimW8A8Int8), this scheme
owns only the hardware-agnostic weight creation; weight post-processing and the
forward pass are delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
This scheme owns weight creation and alignment of TP shards to checkpoint
quantization blocks; weight post-processing and the forward pass are otherwise
delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
process_weights_after_loading detects the pre-quantized float8_e4m3fn weight and
takes the offline (transpose-only) branch.
"""
@@ -13,7 +13,9 @@ takes the offline (transpose-only) branch.
from typing import Dict, List, Optional
import torch
import torch.nn.functional as F
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import (
NPUMXFP8LinearMethod,
)
@@ -60,11 +62,38 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
)
layer.register_parameter("weight", weight)
# msmodelslim exports weight_scale as uint8, shape [out, in/32].
# msmodelslim exports one uint8 scale per 32 input elements. Keep the
# final partial block: Qwen3-VL has visual projections such as K=4304,
# whose checkpoint scale shape is [out, ceil(4304/32)] = [out, 135].
# NOTE: Named "weight_scale" (not "weight_scale_inv") to match the
# checkpoint key exported by msmodelslim; the kernel re-layouts it into
# weight_scale_inv during process_weights_after_loading.
scale_dim = input_size_per_partition // MXFP8_BLOCK_SIZE
scale_dim = (
input_size_per_partition + MXFP8_BLOCK_SIZE - 1
) // MXFP8_BLOCK_SIZE
scale_loader = weight_loader
if (
input_size_per_partition != input_size
and input_size_per_partition % MXFP8_BLOCK_SIZE
):
tp_rank, _ = get_group_rank_size(layer.tp_group)
start = tp_rank * input_size_per_partition
left = start % MXFP8_BLOCK_SIZE
right = -(left + input_size_per_partition) % MXFP8_BLOCK_SIZE
layer.mxfp8_tp_padding = (left, right)
scale_start = start // MXFP8_BLOCK_SIZE
scale_dim = (left + input_size_per_partition + right) // MXFP8_BLOCK_SIZE
# A shard can start inside a checkpoint block, so adjacent ranks
# may share a scale. Include that block and align both W and X to it.
def scale_loader(param, loaded_weight):
if not layer.use_presharded_weights:
loaded_weight = loaded_weight.narrow(1, scale_start, scale_dim)
assert param.shape == loaded_weight.shape, (
f"{param.shape=} {loaded_weight.shape=}"
)
param.data.copy_(loaded_weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
(output_size_per_partition, scale_dim),
@@ -72,11 +101,13 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
weight_loader=scale_loader,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module):
if padding := getattr(layer, "mxfp8_tp_padding", None):
layer.weight.data = F.pad(layer.weight.data, padding)
self.kernel.process_weights_after_loading(layer)
def apply_weights(
@@ -85,4 +116,11 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if padding := getattr(layer, "mxfp8_tp_padding", None):
if isinstance(x, tuple):
raise ValueError(
"ModelSlim MXFP8 with unaligned TP shards requires "
"unquantized inputs for block alignment."
)
x = F.pad(x, padding)
return self.kernel.apply(layer, x, bias)
@@ -300,7 +300,8 @@ class MRotaryEmbedding(RotaryEmbedding):
assert fused_set_kv_buffer_arg is None, (
"fused_set_kv_buffer_arg is not supported for npu implementation"
)
if query.shape[1] > 4096:
# The NPU call below selects ordinary RoPE, which requires 1D positions.
if positions.ndim == 2 or query.shape[1] > 4096:
return self.forward_native(positions, query, key, fused_set_kv_buffer_arg)
rotary_mode = "half" if self.is_neox_style else "interleave"
mrope_section = [0, 0, 0]
+31 -5
View File
@@ -347,6 +347,9 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
if _is_npu:
from sgl_kernel_npu.norm.split_qkv_rmsnorm_mrope import (
triton_split_qkv_rmsnorm_mrope,
)
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import (
split_qkvgate_gemma_rmsnorm_rope,
)
@@ -1541,6 +1544,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
q, k = self.rotary_emb(positions, q, k)
return q, k, v, gate
def forward_prepare_npu_prefill(self, positions, hidden_states):
rotary_emb = self.rotary_emb
if not getattr(rotary_emb, "mrope_section", None):
return self.forward_prepare_native(positions, hidden_states)
qkv, _ = self.qkv_proj(hidden_states)
if positions.ndim == 1:
positions = positions.expand(3, -1)
cos_sin = rotary_emb.cos_sin_cache[positions].to(qkv)
q, k, v, gate = triton_split_qkv_rmsnorm_mrope(
qkv=qkv,
q_weight=self.q_norm.gemma_weight,
k_weight=self.k_norm.gemma_weight,
cos_sin=cos_sin,
num_q_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
eps=self.q_norm.variance_epsilon,
mrope_section=rotary_emb.mrope_section,
is_interleaved=rotary_emb.mrope_interleaved,
rope_dim=rotary_emb.rotary_dim,
has_gate=self.attn_output_gate,
)
return q, k, v, gate if self.attn_output_gate else None
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
qkv, _ = self.qkv_proj(hidden_states)
if self.refreshes_rope_cos_sin:
@@ -1576,11 +1604,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
positions=positions,
hidden_states=hidden_states,
)
if (
not _is_npu
or forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed()
or not self.attn_output_gate
):
if _is_npu and forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed():
return self.forward_prepare_npu_prefill(positions, hidden_states)
if not _is_npu or not self.attn_output_gate:
return self.forward_prepare_native(
positions=positions,
hidden_states=hidden_states,
+103 -1
View File
@@ -1,8 +1,10 @@
import unittest
from unittest.mock import patch
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.layers.rotary_embedding import mrope
from sglang.srt.layers.rotary_embedding.mrope import (
Ernie4_5_VLRotaryEmbedding,
MRotaryEmbedding,
@@ -104,6 +106,106 @@ class TestMRopeAxisMap(CustomTestCase):
)
self.assertIsNone(ernie.axis_map)
def test_npu_dispatch_preserves_multimodal_positions(self):
rope = build_mrope([11, 11, 10], 64, interleaved=True)
rope.head_size = 256
positions = torch.tensor([[0, 1], [2, 3], [4, 5]])
query, key = torch.randn(2, 3072), torch.randn(2, 512)
expected = rope.forward_native(positions, query, key)
with patch.object(mrope, "torch_npu", create=True) as npu:
npu.npu_mrope.return_value = (query, key)
actual = rope.forward_npu(positions, query, key)
npu.npu_mrope.assert_not_called()
torch.testing.assert_close(actual, expected)
text_positions = positions[0]
rope.forward_npu(text_positions, query, key)
npu.npu_mrope.assert_called_once_with(
text_positions,
query,
key,
rope.cos_sin_cache,
256,
mrope_section=[0, 0, 0],
rotary_mode="half",
)
def test_qwen35_npu_prefill_and_decode_dispatch(self):
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.models import qwen3_5
layer = SimpleNamespace(
attn_output_gate=True,
forward_prepare_npu_prefill=Mock(return_value="prefill"),
forward_prepare_npu=Mock(return_value="decode"),
)
with patch.multiple(
qwen3_5,
_is_npu=True,
_is_cuda=False,
_is_hip=False,
_is_cpu=False,
_is_xpu=False,
):
for mode, expected in (
(ForwardMode.EXTEND, "prefill"),
(ForwardMode.DECODE, "decode"),
):
result = qwen3_5.Qwen3_5AttentionDecoderLayer._prepare_qkv_gate(
layer, None, None, SimpleNamespace(forward_mode=mode)
)
self.assertEqual(result, expected)
def test_qwen35_npu_prefill_mrope_contract(self):
from sglang.srt.models import qwen3_5
# Qwen3.5-27B at TP2: 12 Q/gate heads, 2 K/V heads, partial RoPE.
qkv = torch.empty(2, 28 * 256, dtype=torch.bfloat16)
rope = build_mrope([11, 11, 10], 64, interleaved=True)
layer = SimpleNamespace(
qkv_proj=Mock(return_value=(qkv, None)),
rotary_emb=rope,
num_heads=12,
num_kv_heads=2,
head_dim=256,
attn_output_gate=True,
q_norm=SimpleNamespace(
gemma_weight=qkv.new_full((256,), 1.25), variance_epsilon=1e-6
),
k_norm=SimpleNamespace(gemma_weight=qkv.new_full((256,), 0.75)),
)
multimodal = torch.tensor([[0, 1], [2, 3], [4, 5]])
with patch.object(
qwen3_5,
"triton_split_qkv_rmsnorm_mrope",
create=True,
return_value=(None, None, None, None),
) as fused:
for positions in (multimodal, multimodal[0]):
qwen3_5.Qwen3_5AttentionDecoderLayer.forward_prepare_npu_prefill(
layer, positions, None
)
args = fused.call_args.kwargs.copy()
self.assertIs(args.pop("qkv"), qkv)
self.assertIs(args.pop("q_weight"), layer.q_norm.gemma_weight)
self.assertIs(args.pop("k_weight"), layer.k_norm.gemma_weight)
torch.testing.assert_close(
args.pop("cos_sin"),
rope.cos_sin_cache[positions.expand(3, -1)].to(qkv),
)
self.assertEqual(
args,
dict(
num_q_heads=12,
num_kv_heads=2,
head_size=256,
eps=1e-6,
mrope_section=[11, 11, 10],
is_interleaved=True,
rope_dim=64,
has_gate=True,
),
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,179 @@
"""CPU regression tests for ModelSlim MXFP8 weight-scale layouts."""
import unittest
from types import SimpleNamespace
from unittest.mock import Mock
import torch
from sglang.srt.layers.quantization.modelslim.schemes.modelslim_mxfp8 import (
ModelSlimMXFP8Scheme,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
class TestModelSlimMXFP8ScaleLayout(CustomTestCase):
def setUp(self):
super().setUp()
self.scheme = ModelSlimMXFP8Scheme()
def make_layer(self, input_size=4304, tp_size=1, tp_rank=0, presharded=False):
layer = torch.nn.Module()
layer.tp_group = SimpleNamespace(rank_in_group=tp_rank, world_size=tp_size)
layer.use_presharded_weights = presharded
self.scheme.create_weights(
layer,
input_size // tp_size,
[2],
input_size,
2,
torch.bfloat16,
)
return layer
def test_tp_shards_preserve_checkpoint_blocks(self):
k = 4304
weight = (torch.arange(2 * k).reshape(2, k) % 7 - 3).to(torch.float8_e4m3fn)
scales = (torch.arange(2 * 135).reshape(2, 135) % 7 + 124).to(torch.uint8)
reference = (
weight.float()
* torch.exp2(scales.float() - 127).repeat_interleave(32, dim=1)[:, :k]
)
x = (torch.arange(k) % 5 - 2).float().unsqueeze(0)
for tp_size in (1, 2, 4, 8):
result = torch.zeros((1, 2))
for rank in range(tp_size):
with self.subTest(tp_size=tp_size, rank=rank):
layer = self.make_layer(tp_size=tp_size, tp_rank=rank)
start, end = rank * (k // tp_size), (rank + 1) * (k // tp_size)
blocks = torch.arange(start, end) // 32
expected_scales = scales[:, blocks.unique_consecutive()]
if tp_size == 1:
layer.weight_scale.data.copy_(scales)
else:
layer.weight_scale.weight_loader(layer.weight_scale, scales)
torch.testing.assert_close(layer.weight_scale, expected_scales)
if tp_size == 4 and rank in (1, 3):
self.assertEqual(layer.weight_scale.shape[1], 35)
layer.weight.data.copy_(weight[:, start:end])
self.scheme.process_weights_after_loading(layer)
# Undo the kernel's paired scale layout and compare every
# real weight element against the global checkpoint blocks.
local_scales = layer.weight_scale_inv.transpose(0, 1).reshape(2, -1)
local_weight = (
layer.weight.t().float()
* torch.exp2(local_scales.float() - 127).repeat_interleave(
32, dim=1
)[:, : layer.weight.shape[0]]
)
left, right = getattr(layer, "mxfp8_tp_padding", (0, 0))
torch.testing.assert_close(
local_weight[:, left : left + end - start],
reference[:, start:end],
)
self.assertEqual(torch.count_nonzero(local_weight[:, :left]), 0)
if right:
self.assertEqual(
torch.count_nonzero(local_weight[:, -right:]), 0
)
padded_x = torch.nn.functional.pad(x[:, start:end], (left, right))
result += padded_x @ local_weight.t()
torch.testing.assert_close(result, x @ reference.t())
def test_presharded_scales_are_loaded_without_reslicing(self):
layer = self.make_layer(tp_size=4, tp_rank=3, presharded=True)
scales = torch.arange(70, dtype=torch.uint8).reshape(2, 35)
layer.weight_scale.weight_loader(layer.weight_scale, scales)
torch.testing.assert_close(layer.weight_scale, scales)
with self.assertRaises(AssertionError):
layer.weight_scale.weight_loader(layer.weight_scale, scales[:, :-1])
def test_unaligned_input_padding_precedes_kernel(self):
layer = self.make_layer(tp_size=4, tp_rank=3)
self.scheme.kernel = Mock()
for shape in ((2, 1076), (2, 3, 1076)):
with self.subTest(shape=shape):
x = torch.ones(shape, dtype=torch.bfloat16)
bias = torch.ones(2)
result = self.scheme.apply_weights(layer, x, bias)
self.assertIs(result, self.scheme.kernel.apply.return_value)
padded = self.scheme.kernel.apply.call_args.args[1]
torch.testing.assert_close(padded, torch.nn.functional.pad(x, (28, 16)))
self.assertIs(self.scheme.kernel.apply.call_args.args[2], bias)
self.scheme.kernel.reset_mock()
with self.assertRaisesRegex(ValueError, "unquantized inputs"):
self.scheme.apply_weights(layer, (torch.ones(1), torch.ones(1)))
self.scheme.kernel.apply.assert_not_called()
def test_unsharded_column_and_aligned_paths_keep_original_loader(self):
for local_k, full_k, output_sizes, output_size in (
(4304, 4304, [2], 2), # TP1, partial final block.
(4304, 4304, [1], 2), # Column-parallel partition.
(96, 192, [2], 2), # Row-parallel, odd but aligned scale count.
):
with self.subTest(local_k=local_k, output_sizes=output_sizes):
layer = torch.nn.Module()
loader = Mock()
self.scheme.create_weights(
layer,
local_k,
output_sizes,
full_k,
output_size,
torch.bfloat16,
weight_loader=loader,
)
self.assertIs(layer.weight_scale.weight_loader, loader)
self.assertIs(layer.weight.weight_loader, loader)
self.assertFalse(hasattr(layer, "mxfp8_tp_padding"))
self.scheme.kernel = Mock()
for x in (torch.ones(2, local_k), (torch.ones(1), torch.ones(1))):
self.scheme.apply_weights(layer, x)
self.assertIs(self.scheme.kernel.apply.call_args.args[1], x)
def test_scale_placeholder_rounds_up_partial_block(self):
layer = torch.nn.Module()
self.scheme.create_weights(
layer=layer,
input_size_per_partition=4304,
output_partition_sizes=[2],
input_size=4304,
output_size=2,
params_dtype=torch.bfloat16,
)
self.assertEqual(layer.weight_scale.shape, (2, 135))
def test_post_load_pads_odd_scale_count_for_pair_layout(self):
layer = torch.nn.Module()
layer.register_parameter(
"weight",
torch.nn.Parameter(
torch.empty((2, 4304), dtype=torch.float8_e4m3fn),
requires_grad=False,
),
)
layer.register_parameter(
"weight_scale",
torch.nn.Parameter(
torch.ones((2, 135), dtype=torch.uint8), requires_grad=False
),
)
layer.register_parameter("bias", None)
self.scheme.process_weights_after_loading(layer)
self.assertEqual(layer.weight.shape, (4304, 2))
self.assertEqual(layer.weight_scale_inv.shape, (68, 2, 2))
self.assertTrue(torch.all(layer.weight_scale_inv[-1, :, 1] == 0))
self.assertFalse(hasattr(layer, "weight_scale"))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,107 @@
import re
import sys
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
@pytest.mark.parametrize(
("layer_name", "eps_attr"),
[("GemmaRMSNorm", "variance_epsilon"), ("Gemma3RMSNorm", "eps")],
)
def test_srt_gemma_layers_delegate_plain_npu_path(layer_name, eps_attr):
from sglang.srt.layers import layernorm as layernorm_module
layer = getattr(layernorm_module, layer_name)(4)
x = torch.randn(2, 4)
unified_op = MagicMock(return_value=(x, "rstd"))
with patch.object(layernorm_module, "npu_gemma_rms_norm", unified_op, create=True):
result = layer.forward_npu(x)
assert result is x
unified_op.assert_called_once_with(x, layer.weight, getattr(layer, eps_attr))
@pytest.mark.parametrize("hidden_size", [256, 5120, 8192])
@pytest.mark.parametrize(
("weight_dtype", "input_dtype"),
[
(torch.float16, torch.float16),
(torch.bfloat16, torch.bfloat16),
(torch.float32, torch.bfloat16),
],
)
@pytest.mark.parametrize("with_post_residual", [False, True])
def test_srt_gemma_residual_reuses_precomputed_weight(
hidden_size, weight_dtype, input_dtype, with_post_residual
):
from sglang.srt.layers import layernorm as layernorm_module
layer = layernorm_module.GemmaRMSNorm(hidden_size).to(weight_dtype)
weight = torch.linspace(-0.5, 0.5, hidden_size, dtype=weight_dtype)
layer.weight.weight_loader(layer.weight, weight)
x = torch.randn(2, hidden_size, dtype=input_dtype)
residual = torch.randn_like(x)
post_residual = torch.randn_like(x) if with_post_residual else None
output, residual_sum = torch.empty_like(x), torch.empty_like(x)
fused_op = MagicMock(return_value=(output, None, residual_sum))
with patch.object(
layernorm_module,
"torch_npu",
SimpleNamespace(npu_add_rms_norm=fused_op),
create=True,
):
result = layer.forward_npu(x, residual, post_residual)
assert result[0] is output
assert result[1] is residual_sum
fused_op.assert_called_once()
actual_residual, actual_x, gamma, eps = fused_op.call_args.args
assert actual_x is x
assert eps == layer.variance_epsilon
if with_post_residual:
torch.testing.assert_close(actual_residual, residual + post_residual)
else:
assert actual_residual is residual
assert gamma.dtype == input_dtype
torch.testing.assert_close(gamma, (1.0 + weight).to(input_dtype))
if weight_dtype == input_dtype:
assert gamma is layer.gemma_weight
def test_srt_falls_back_to_torch_npu_on_wheels_without_the_provider():
"""The srt-layer provider import must stay guarded, with a torch_npu fallback.
``srt/layers/layernorm.py`` imports the provider at module scope under
``if _is_npu``, which CPU CI never executes -- hence a source-shape check.
Turning it back into a hard import would break every NPU deployment running
an sgl-kernel-npu wheel from before the staged Gemma provider, at
``import sglang.srt.layers.layernorm`` time and for all models, not just
Gemma ones.
"""
from sglang.srt.layers import layernorm as layernorm_module
source = Path(layernorm_module.__file__).read_text(encoding="utf-8")
guarded_import = re.search(
r"try:\s*\n"
r"\s*from sgl_kernel_npu\.norm\.gemma_rmsnorm import npu_gemma_rms_norm\s*\n"
r"\s*except ImportError:\s*\n"
r"(?:\s*#.*\n)*"
r"\s*npu_gemma_rms_norm = torch_npu\.npu_gemma_rms_norm\s*\n",
source,
)
assert guarded_import is not None
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))