From 844d2b5decf6257aa192744cbdaaffb8ad797263 Mon Sep 17 00:00:00 2001 From: Junlin Wu Date: Wed, 21 Oct 2026 14:50:38 +0800 Subject: [PATCH] [Fix] [NPU] qwen3.5: Fix dense serving on Ascend 950 (#32745) --- .../getting-started/installation.mdx | 12 ++ .../npu/quantization/linear_method_npu.py | 12 +- python/sglang/srt/layers/layernorm.py | 27 +-- .../modelslim/schemes/modelslim_mxfp8.py | 50 ++++- .../srt/layers/rotary_embedding/mrope.py | 3 +- python/sglang/srt/models/qwen3_5.py | 36 +++- test/registered/rotary/test_mrope_axis_map.py | 104 +++++++++- .../quantization/test_modelslim_mxfp8.py | 179 ++++++++++++++++++ .../unit/npu/test_npu_gemma_rmsnorm.py | 107 +++++++++++ 9 files changed, 501 insertions(+), 29 deletions(-) create mode 100644 test/registered/unit/layers/quantization/test_modelslim_mxfp8.py create mode 100644 test/registered/unit/npu/test_npu_gemma_rmsnorm.py diff --git a/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx b/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx index 17de0361af..21e94bb0e7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/getting-started/installation.mdx @@ -245,6 +245,18 @@ For installation of Triton on Ascend nightly builds or from sources, follow [ins We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md). + +Gemma-family models (including Qwen3.5) call `sgl_kernel_npu.norm.gemma_rmsnorm`, +whose implementation is chosen when the wheel is built: the native `torch_npu` +operator for Ascend 910B/910C, and ACLNN RMSNorm for Ascend 950, which does not +register that operator. **Ascend 950 users must install the `950` release +package** (`950` appears in the asset name), or build from source with +`bash build.sh -a kernels Ascend950PR_9599`, the target that package is built +with. On Ascend 950, a `910b` or `a3` package fails on the first Gemma forward, +and so does a package from a release that predates this module; check with +`python -c "import sgl_kernel_npu.norm.gemma_rmsnorm"`. + + #### DeepEP-compatible Library We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). diff --git a/python/sglang/srt/hardware_backend/npu/quantization/linear_method_npu.py b/python/sglang/srt/hardware_backend/npu/quantization/linear_method_npu.py index 3c905bf80d..b2dab4d53a 100644 --- a/python/sglang/srt/hardware_backend/npu/quantization/linear_method_npu.py +++ b/python/sglang/srt/hardware_backend/npu/quantization/linear_method_npu.py @@ -2,6 +2,7 @@ import logging from typing import TYPE_CHECKING, List, Optional, Tuple import torch +import torch.nn.functional as F from torch.nn.parameter import Parameter from sglang.srt.hardware_backend.npu.utils import NPUACLFormat, npu_format_cast @@ -204,10 +205,15 @@ class NPUMXFP8LinearMethod(_NPULinearMethodBase): weight = layer.weight.data if weight.dtype == torch.float8_e4m3fn: # Offline (ModelSlim) path: weight is already MXFP8-quantised and - # layer.weight_scale holds the uint8 block scales [out, in/32]. Only - # re-layout to [in, out] / [in//64, out, 2] strided views below. + # layer.weight_scale holds one uint8 scale per 32 input elements. + # Pair the scales for the NPU kernel, padding an odd final count + # (e.g. K=4304 -> 135 scales -> 136 -> 68 pairs). n_dim, k_dim = layer.weight_scale.data.shape - scale = layer.weight_scale.data.reshape(n_dim, k_dim // 2, 2) + scale_data = layer.weight_scale.data + if k_dim % 2 != 0: + scale_data = F.pad(scale_data, (0, 1), mode="constant", value=0) + k_dim += 1 + scale = scale_data.reshape(n_dim, k_dim // 2, 2) layer.weight = Parameter(weight.transpose(0, 1), requires_grad=False) layer.weight_scale_inv = Parameter( scale.transpose(0, 1), requires_grad=False diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 67d8f8f76c..ec64ba55fb 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -182,9 +182,16 @@ logger = logging.getLogger(__name__) if _is_npu: import torch_npu - from sgl_kernel_npu.norm.add_rmsnorm_bias import add_gemma_rms_norm -_NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE = 5120 + try: + from sgl_kernel_npu.norm.gemma_rmsnorm import npu_gemma_rms_norm + except ImportError: + # sgl-kernel-npu wheels built before the target-specific Gemma provider + # landed expose only the torch_npu operator, which is exactly what that + # provider binds to on A2/A3 — so those deployments keep working + # unchanged. On Ascend 950 the operator is unregistered and the first Gemma + # forward fails loudly, same as before this indirection existed. + npu_gemma_rms_norm = torch_npu.npu_gemma_rms_norm @lru_cache(maxsize=1) @@ -1237,18 +1244,12 @@ class GemmaRMSNorm(BaseFusedOp): if residual is not None: if post_residual_addition is not None: residual = residual + post_residual_addition - if x.shape[-1] > _NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE: - gamma = self.gemma_weight.to(x.dtype) - norm_out, _, residual = torch_npu.npu_add_rms_norm( - residual, x, gamma, self.variance_epsilon - ) - else: - norm_out, residual = add_gemma_rms_norm( - x, self.weight, residual, self.variance_epsilon - ) + norm_out, _, residual = torch_npu.npu_add_rms_norm( + residual, x, self.gemma_weight.to(x.dtype), self.variance_epsilon + ) return norm_out, residual - x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon) + x, _ = npu_gemma_rms_norm(x, self.weight, self.variance_epsilon) return x def forward_xpu( @@ -1376,7 +1377,7 @@ class Gemma3RMSNorm(BaseFusedOp): return self.forward_native(x, residual) if residual is not None: return self.forward_native(x, residual) - output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps) + output, _ = npu_gemma_rms_norm(x, self.weight, self.eps) return output def extra_repr(self): diff --git a/python/sglang/srt/layers/quantization/modelslim/schemes/modelslim_mxfp8.py b/python/sglang/srt/layers/quantization/modelslim/schemes/modelslim_mxfp8.py index 8c82ed9329..42b9579554 100644 --- a/python/sglang/srt/layers/quantization/modelslim/schemes/modelslim_mxfp8.py +++ b/python/sglang/srt/layers/quantization/modelslim/schemes/modelslim_mxfp8.py @@ -3,9 +3,9 @@ Loads weights pre-quantized by msmodelslim (float8_e4m3fn weights, uint8 scales) and runs MXFP8 matmul at inference. -Following the modelslim-scheme convention (see ModelSlimW8A8Int8), this scheme -owns only the hardware-agnostic weight creation; weight post-processing and the -forward pass are delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its +This scheme owns weight creation and alignment of TP shards to checkpoint +quantization blocks; weight post-processing and the forward pass are otherwise +delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its process_weights_after_loading detects the pre-quantized float8_e4m3fn weight and takes the offline (transpose-only) branch. """ @@ -13,7 +13,9 @@ takes the offline (transpose-only) branch. from typing import Dict, List, Optional import torch +import torch.nn.functional as F +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import ( NPUMXFP8LinearMethod, ) @@ -60,11 +62,38 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme): ) layer.register_parameter("weight", weight) - # msmodelslim exports weight_scale as uint8, shape [out, in/32]. + # msmodelslim exports one uint8 scale per 32 input elements. Keep the + # final partial block: Qwen3-VL has visual projections such as K=4304, + # whose checkpoint scale shape is [out, ceil(4304/32)] = [out, 135]. # NOTE: Named "weight_scale" (not "weight_scale_inv") to match the # checkpoint key exported by msmodelslim; the kernel re-layouts it into # weight_scale_inv during process_weights_after_loading. - scale_dim = input_size_per_partition // MXFP8_BLOCK_SIZE + scale_dim = ( + input_size_per_partition + MXFP8_BLOCK_SIZE - 1 + ) // MXFP8_BLOCK_SIZE + scale_loader = weight_loader + if ( + input_size_per_partition != input_size + and input_size_per_partition % MXFP8_BLOCK_SIZE + ): + tp_rank, _ = get_group_rank_size(layer.tp_group) + start = tp_rank * input_size_per_partition + left = start % MXFP8_BLOCK_SIZE + right = -(left + input_size_per_partition) % MXFP8_BLOCK_SIZE + layer.mxfp8_tp_padding = (left, right) + scale_start = start // MXFP8_BLOCK_SIZE + scale_dim = (left + input_size_per_partition + right) // MXFP8_BLOCK_SIZE + + # A shard can start inside a checkpoint block, so adjacent ranks + # may share a scale. Include that block and align both W and X to it. + def scale_loader(param, loaded_weight): + if not layer.use_presharded_weights: + loaded_weight = loaded_weight.narrow(1, scale_start, scale_dim) + assert param.shape == loaded_weight.shape, ( + f"{param.shape=} {loaded_weight.shape=}" + ) + param.data.copy_(loaded_weight) + weight_scale = GroupQuantScaleParameter( data=torch.empty( (output_size_per_partition, scale_dim), @@ -72,11 +101,13 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme): ), input_dim=1, output_dim=0, - weight_loader=weight_loader, + weight_loader=scale_loader, ) layer.register_parameter("weight_scale", weight_scale) def process_weights_after_loading(self, layer: torch.nn.Module): + if padding := getattr(layer, "mxfp8_tp_padding", None): + layer.weight.data = F.pad(layer.weight.data, padding) self.kernel.process_weights_after_loading(layer) def apply_weights( @@ -85,4 +116,11 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme): x: torch.Tensor, bias: Optional[torch.Tensor] = None, ) -> torch.Tensor: + if padding := getattr(layer, "mxfp8_tp_padding", None): + if isinstance(x, tuple): + raise ValueError( + "ModelSlim MXFP8 with unaligned TP shards requires " + "unquantized inputs for block alignment." + ) + x = F.pad(x, padding) return self.kernel.apply(layer, x, bias) diff --git a/python/sglang/srt/layers/rotary_embedding/mrope.py b/python/sglang/srt/layers/rotary_embedding/mrope.py index 32bba1936e..b6f19828af 100644 --- a/python/sglang/srt/layers/rotary_embedding/mrope.py +++ b/python/sglang/srt/layers/rotary_embedding/mrope.py @@ -300,7 +300,8 @@ class MRotaryEmbedding(RotaryEmbedding): assert fused_set_kv_buffer_arg is None, ( "fused_set_kv_buffer_arg is not supported for npu implementation" ) - if query.shape[1] > 4096: + # The NPU call below selects ordinary RoPE, which requires 1D positions. + if positions.ndim == 2 or query.shape[1] > 4096: return self.forward_native(positions, query, key, fused_set_kv_buffer_arg) rotary_mode = "half" if self.is_neox_style else "interleave" mrope_section = [0, 0, 0] diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 5c9dfb448e..5e452cb6b2 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -347,6 +347,9 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module): if _is_npu: + from sgl_kernel_npu.norm.split_qkv_rmsnorm_mrope import ( + triton_split_qkv_rmsnorm_mrope, + ) from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import ( split_qkvgate_gemma_rmsnorm_rope, ) @@ -1541,6 +1544,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): q, k = self.rotary_emb(positions, q, k) return q, k, v, gate + def forward_prepare_npu_prefill(self, positions, hidden_states): + rotary_emb = self.rotary_emb + if not getattr(rotary_emb, "mrope_section", None): + return self.forward_prepare_native(positions, hidden_states) + + qkv, _ = self.qkv_proj(hidden_states) + if positions.ndim == 1: + positions = positions.expand(3, -1) + cos_sin = rotary_emb.cos_sin_cache[positions].to(qkv) + q, k, v, gate = triton_split_qkv_rmsnorm_mrope( + qkv=qkv, + q_weight=self.q_norm.gemma_weight, + k_weight=self.k_norm.gemma_weight, + cos_sin=cos_sin, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + eps=self.q_norm.variance_epsilon, + mrope_section=rotary_emb.mrope_section, + is_interleaved=rotary_emb.mrope_interleaved, + rope_dim=rotary_emb.rotary_dim, + has_gate=self.attn_output_gate, + ) + return q, k, v, gate if self.attn_output_gate else None + def forward_prepare_npu(self, positions, hidden_states, forward_batch): qkv, _ = self.qkv_proj(hidden_states) if self.refreshes_rope_cos_sin: @@ -1576,11 +1604,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): positions=positions, hidden_states=hidden_states, ) - if ( - not _is_npu - or forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed() - or not self.attn_output_gate - ): + if _is_npu and forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed(): + return self.forward_prepare_npu_prefill(positions, hidden_states) + if not _is_npu or not self.attn_output_gate: return self.forward_prepare_native( positions=positions, hidden_states=hidden_states, diff --git a/test/registered/rotary/test_mrope_axis_map.py b/test/registered/rotary/test_mrope_axis_map.py index 3d60aac50f..0a1f5f4711 100644 --- a/test/registered/rotary/test_mrope_axis_map.py +++ b/test/registered/rotary/test_mrope_axis_map.py @@ -1,8 +1,10 @@ import unittest -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import Mock, patch import torch +from sglang.srt.layers.rotary_embedding import mrope from sglang.srt.layers.rotary_embedding.mrope import ( Ernie4_5_VLRotaryEmbedding, MRotaryEmbedding, @@ -104,6 +106,106 @@ class TestMRopeAxisMap(CustomTestCase): ) self.assertIsNone(ernie.axis_map) + def test_npu_dispatch_preserves_multimodal_positions(self): + rope = build_mrope([11, 11, 10], 64, interleaved=True) + rope.head_size = 256 + positions = torch.tensor([[0, 1], [2, 3], [4, 5]]) + query, key = torch.randn(2, 3072), torch.randn(2, 512) + expected = rope.forward_native(positions, query, key) + with patch.object(mrope, "torch_npu", create=True) as npu: + npu.npu_mrope.return_value = (query, key) + actual = rope.forward_npu(positions, query, key) + npu.npu_mrope.assert_not_called() + torch.testing.assert_close(actual, expected) + text_positions = positions[0] + rope.forward_npu(text_positions, query, key) + npu.npu_mrope.assert_called_once_with( + text_positions, + query, + key, + rope.cos_sin_cache, + 256, + mrope_section=[0, 0, 0], + rotary_mode="half", + ) + + def test_qwen35_npu_prefill_and_decode_dispatch(self): + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models import qwen3_5 + + layer = SimpleNamespace( + attn_output_gate=True, + forward_prepare_npu_prefill=Mock(return_value="prefill"), + forward_prepare_npu=Mock(return_value="decode"), + ) + with patch.multiple( + qwen3_5, + _is_npu=True, + _is_cuda=False, + _is_hip=False, + _is_cpu=False, + _is_xpu=False, + ): + for mode, expected in ( + (ForwardMode.EXTEND, "prefill"), + (ForwardMode.DECODE, "decode"), + ): + result = qwen3_5.Qwen3_5AttentionDecoderLayer._prepare_qkv_gate( + layer, None, None, SimpleNamespace(forward_mode=mode) + ) + self.assertEqual(result, expected) + + def test_qwen35_npu_prefill_mrope_contract(self): + from sglang.srt.models import qwen3_5 + + # Qwen3.5-27B at TP2: 12 Q/gate heads, 2 K/V heads, partial RoPE. + qkv = torch.empty(2, 28 * 256, dtype=torch.bfloat16) + rope = build_mrope([11, 11, 10], 64, interleaved=True) + layer = SimpleNamespace( + qkv_proj=Mock(return_value=(qkv, None)), + rotary_emb=rope, + num_heads=12, + num_kv_heads=2, + head_dim=256, + attn_output_gate=True, + q_norm=SimpleNamespace( + gemma_weight=qkv.new_full((256,), 1.25), variance_epsilon=1e-6 + ), + k_norm=SimpleNamespace(gemma_weight=qkv.new_full((256,), 0.75)), + ) + multimodal = torch.tensor([[0, 1], [2, 3], [4, 5]]) + with patch.object( + qwen3_5, + "triton_split_qkv_rmsnorm_mrope", + create=True, + return_value=(None, None, None, None), + ) as fused: + for positions in (multimodal, multimodal[0]): + qwen3_5.Qwen3_5AttentionDecoderLayer.forward_prepare_npu_prefill( + layer, positions, None + ) + args = fused.call_args.kwargs.copy() + self.assertIs(args.pop("qkv"), qkv) + self.assertIs(args.pop("q_weight"), layer.q_norm.gemma_weight) + self.assertIs(args.pop("k_weight"), layer.k_norm.gemma_weight) + torch.testing.assert_close( + args.pop("cos_sin"), + rope.cos_sin_cache[positions.expand(3, -1)].to(qkv), + ) + self.assertEqual( + args, + dict( + num_q_heads=12, + num_kv_heads=2, + head_size=256, + eps=1e-6, + mrope_section=[11, 11, 10], + is_interleaved=True, + rope_dim=64, + has_gate=True, + ), + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/quantization/test_modelslim_mxfp8.py b/test/registered/unit/layers/quantization/test_modelslim_mxfp8.py new file mode 100644 index 0000000000..bb9f6e895e --- /dev/null +++ b/test/registered/unit/layers/quantization/test_modelslim_mxfp8.py @@ -0,0 +1,179 @@ +"""CPU regression tests for ModelSlim MXFP8 weight-scale layouts.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from sglang.srt.layers.quantization.modelslim.schemes.modelslim_mxfp8 import ( + ModelSlimMXFP8Scheme, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +class TestModelSlimMXFP8ScaleLayout(CustomTestCase): + def setUp(self): + super().setUp() + self.scheme = ModelSlimMXFP8Scheme() + + def make_layer(self, input_size=4304, tp_size=1, tp_rank=0, presharded=False): + layer = torch.nn.Module() + layer.tp_group = SimpleNamespace(rank_in_group=tp_rank, world_size=tp_size) + layer.use_presharded_weights = presharded + self.scheme.create_weights( + layer, + input_size // tp_size, + [2], + input_size, + 2, + torch.bfloat16, + ) + return layer + + def test_tp_shards_preserve_checkpoint_blocks(self): + k = 4304 + weight = (torch.arange(2 * k).reshape(2, k) % 7 - 3).to(torch.float8_e4m3fn) + scales = (torch.arange(2 * 135).reshape(2, 135) % 7 + 124).to(torch.uint8) + reference = ( + weight.float() + * torch.exp2(scales.float() - 127).repeat_interleave(32, dim=1)[:, :k] + ) + x = (torch.arange(k) % 5 - 2).float().unsqueeze(0) + + for tp_size in (1, 2, 4, 8): + result = torch.zeros((1, 2)) + for rank in range(tp_size): + with self.subTest(tp_size=tp_size, rank=rank): + layer = self.make_layer(tp_size=tp_size, tp_rank=rank) + start, end = rank * (k // tp_size), (rank + 1) * (k // tp_size) + blocks = torch.arange(start, end) // 32 + expected_scales = scales[:, blocks.unique_consecutive()] + if tp_size == 1: + layer.weight_scale.data.copy_(scales) + else: + layer.weight_scale.weight_loader(layer.weight_scale, scales) + torch.testing.assert_close(layer.weight_scale, expected_scales) + if tp_size == 4 and rank in (1, 3): + self.assertEqual(layer.weight_scale.shape[1], 35) + layer.weight.data.copy_(weight[:, start:end]) + self.scheme.process_weights_after_loading(layer) + + # Undo the kernel's paired scale layout and compare every + # real weight element against the global checkpoint blocks. + local_scales = layer.weight_scale_inv.transpose(0, 1).reshape(2, -1) + local_weight = ( + layer.weight.t().float() + * torch.exp2(local_scales.float() - 127).repeat_interleave( + 32, dim=1 + )[:, : layer.weight.shape[0]] + ) + left, right = getattr(layer, "mxfp8_tp_padding", (0, 0)) + torch.testing.assert_close( + local_weight[:, left : left + end - start], + reference[:, start:end], + ) + self.assertEqual(torch.count_nonzero(local_weight[:, :left]), 0) + if right: + self.assertEqual( + torch.count_nonzero(local_weight[:, -right:]), 0 + ) + padded_x = torch.nn.functional.pad(x[:, start:end], (left, right)) + result += padded_x @ local_weight.t() + torch.testing.assert_close(result, x @ reference.t()) + + def test_presharded_scales_are_loaded_without_reslicing(self): + layer = self.make_layer(tp_size=4, tp_rank=3, presharded=True) + scales = torch.arange(70, dtype=torch.uint8).reshape(2, 35) + layer.weight_scale.weight_loader(layer.weight_scale, scales) + torch.testing.assert_close(layer.weight_scale, scales) + with self.assertRaises(AssertionError): + layer.weight_scale.weight_loader(layer.weight_scale, scales[:, :-1]) + + def test_unaligned_input_padding_precedes_kernel(self): + layer = self.make_layer(tp_size=4, tp_rank=3) + self.scheme.kernel = Mock() + for shape in ((2, 1076), (2, 3, 1076)): + with self.subTest(shape=shape): + x = torch.ones(shape, dtype=torch.bfloat16) + bias = torch.ones(2) + result = self.scheme.apply_weights(layer, x, bias) + self.assertIs(result, self.scheme.kernel.apply.return_value) + padded = self.scheme.kernel.apply.call_args.args[1] + torch.testing.assert_close(padded, torch.nn.functional.pad(x, (28, 16))) + self.assertIs(self.scheme.kernel.apply.call_args.args[2], bias) + self.scheme.kernel.reset_mock() + with self.assertRaisesRegex(ValueError, "unquantized inputs"): + self.scheme.apply_weights(layer, (torch.ones(1), torch.ones(1))) + self.scheme.kernel.apply.assert_not_called() + + def test_unsharded_column_and_aligned_paths_keep_original_loader(self): + for local_k, full_k, output_sizes, output_size in ( + (4304, 4304, [2], 2), # TP1, partial final block. + (4304, 4304, [1], 2), # Column-parallel partition. + (96, 192, [2], 2), # Row-parallel, odd but aligned scale count. + ): + with self.subTest(local_k=local_k, output_sizes=output_sizes): + layer = torch.nn.Module() + loader = Mock() + self.scheme.create_weights( + layer, + local_k, + output_sizes, + full_k, + output_size, + torch.bfloat16, + weight_loader=loader, + ) + self.assertIs(layer.weight_scale.weight_loader, loader) + self.assertIs(layer.weight.weight_loader, loader) + self.assertFalse(hasattr(layer, "mxfp8_tp_padding")) + self.scheme.kernel = Mock() + for x in (torch.ones(2, local_k), (torch.ones(1), torch.ones(1))): + self.scheme.apply_weights(layer, x) + self.assertIs(self.scheme.kernel.apply.call_args.args[1], x) + + def test_scale_placeholder_rounds_up_partial_block(self): + layer = torch.nn.Module() + + self.scheme.create_weights( + layer=layer, + input_size_per_partition=4304, + output_partition_sizes=[2], + input_size=4304, + output_size=2, + params_dtype=torch.bfloat16, + ) + + self.assertEqual(layer.weight_scale.shape, (2, 135)) + + def test_post_load_pads_odd_scale_count_for_pair_layout(self): + layer = torch.nn.Module() + layer.register_parameter( + "weight", + torch.nn.Parameter( + torch.empty((2, 4304), dtype=torch.float8_e4m3fn), + requires_grad=False, + ), + ) + layer.register_parameter( + "weight_scale", + torch.nn.Parameter( + torch.ones((2, 135), dtype=torch.uint8), requires_grad=False + ), + ) + layer.register_parameter("bias", None) + + self.scheme.process_weights_after_loading(layer) + + self.assertEqual(layer.weight.shape, (4304, 2)) + self.assertEqual(layer.weight_scale_inv.shape, (68, 2, 2)) + self.assertTrue(torch.all(layer.weight_scale_inv[-1, :, 1] == 0)) + self.assertFalse(hasattr(layer, "weight_scale")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/npu/test_npu_gemma_rmsnorm.py b/test/registered/unit/npu/test_npu_gemma_rmsnorm.py new file mode 100644 index 0000000000..aef80bf7e3 --- /dev/null +++ b/test/registered/unit/npu/test_npu_gemma_rmsnorm.py @@ -0,0 +1,107 @@ +import re +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +@pytest.mark.parametrize( + ("layer_name", "eps_attr"), + [("GemmaRMSNorm", "variance_epsilon"), ("Gemma3RMSNorm", "eps")], +) +def test_srt_gemma_layers_delegate_plain_npu_path(layer_name, eps_attr): + from sglang.srt.layers import layernorm as layernorm_module + + layer = getattr(layernorm_module, layer_name)(4) + x = torch.randn(2, 4) + unified_op = MagicMock(return_value=(x, "rstd")) + + with patch.object(layernorm_module, "npu_gemma_rms_norm", unified_op, create=True): + result = layer.forward_npu(x) + + assert result is x + unified_op.assert_called_once_with(x, layer.weight, getattr(layer, eps_attr)) + + +@pytest.mark.parametrize("hidden_size", [256, 5120, 8192]) +@pytest.mark.parametrize( + ("weight_dtype", "input_dtype"), + [ + (torch.float16, torch.float16), + (torch.bfloat16, torch.bfloat16), + (torch.float32, torch.bfloat16), + ], +) +@pytest.mark.parametrize("with_post_residual", [False, True]) +def test_srt_gemma_residual_reuses_precomputed_weight( + hidden_size, weight_dtype, input_dtype, with_post_residual +): + from sglang.srt.layers import layernorm as layernorm_module + + layer = layernorm_module.GemmaRMSNorm(hidden_size).to(weight_dtype) + weight = torch.linspace(-0.5, 0.5, hidden_size, dtype=weight_dtype) + layer.weight.weight_loader(layer.weight, weight) + x = torch.randn(2, hidden_size, dtype=input_dtype) + residual = torch.randn_like(x) + post_residual = torch.randn_like(x) if with_post_residual else None + output, residual_sum = torch.empty_like(x), torch.empty_like(x) + fused_op = MagicMock(return_value=(output, None, residual_sum)) + + with patch.object( + layernorm_module, + "torch_npu", + SimpleNamespace(npu_add_rms_norm=fused_op), + create=True, + ): + result = layer.forward_npu(x, residual, post_residual) + + assert result[0] is output + assert result[1] is residual_sum + fused_op.assert_called_once() + actual_residual, actual_x, gamma, eps = fused_op.call_args.args + assert actual_x is x + assert eps == layer.variance_epsilon + if with_post_residual: + torch.testing.assert_close(actual_residual, residual + post_residual) + else: + assert actual_residual is residual + assert gamma.dtype == input_dtype + torch.testing.assert_close(gamma, (1.0 + weight).to(input_dtype)) + if weight_dtype == input_dtype: + assert gamma is layer.gemma_weight + + +def test_srt_falls_back_to_torch_npu_on_wheels_without_the_provider(): + """The srt-layer provider import must stay guarded, with a torch_npu fallback. + + ``srt/layers/layernorm.py`` imports the provider at module scope under + ``if _is_npu``, which CPU CI never executes -- hence a source-shape check. + Turning it back into a hard import would break every NPU deployment running + an sgl-kernel-npu wheel from before the staged Gemma provider, at + ``import sglang.srt.layers.layernorm`` time and for all models, not just + Gemma ones. + """ + from sglang.srt.layers import layernorm as layernorm_module + + source = Path(layernorm_module.__file__).read_text(encoding="utf-8") + guarded_import = re.search( + r"try:\s*\n" + r"\s*from sgl_kernel_npu\.norm\.gemma_rmsnorm import npu_gemma_rms_norm\s*\n" + r"\s*except ImportError:\s*\n" + r"(?:\s*#.*\n)*" + r"\s*npu_gemma_rms_norm = torch_npu\.npu_gemma_rms_norm\s*\n", + source, + ) + + assert guarded_import is not None + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))