[Fix] [NPU] qwen3.5: Fix dense serving on Ascend 950 (#32745)
This commit is contained in:
@@ -245,6 +245,18 @@ For installation of Triton on Ascend nightly builds or from sources, follow [ins
|
||||
|
||||
We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md).
|
||||
|
||||
<Note>
|
||||
Gemma-family models (including Qwen3.5) call `sgl_kernel_npu.norm.gemma_rmsnorm`,
|
||||
whose implementation is chosen when the wheel is built: the native `torch_npu`
|
||||
operator for Ascend 910B/910C, and ACLNN RMSNorm for Ascend 950, which does not
|
||||
register that operator. **Ascend 950 users must install the `950` release
|
||||
package** (`950` appears in the asset name), or build from source with
|
||||
`bash build.sh -a kernels Ascend950PR_9599`, the target that package is built
|
||||
with. On Ascend 950, a `910b` or `a3` package fails on the first Gemma forward,
|
||||
and so does a package from a release that predates this module; check with
|
||||
`python -c "import sgl_kernel_npu.norm.gemma_rmsnorm"`.
|
||||
</Note>
|
||||
|
||||
#### DeepEP-compatible Library
|
||||
|
||||
We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md).
|
||||
|
||||
@@ -2,6 +2,7 @@ import logging
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from sglang.srt.hardware_backend.npu.utils import NPUACLFormat, npu_format_cast
|
||||
@@ -204,10 +205,15 @@ class NPUMXFP8LinearMethod(_NPULinearMethodBase):
|
||||
weight = layer.weight.data
|
||||
if weight.dtype == torch.float8_e4m3fn:
|
||||
# Offline (ModelSlim) path: weight is already MXFP8-quantised and
|
||||
# layer.weight_scale holds the uint8 block scales [out, in/32]. Only
|
||||
# re-layout to [in, out] / [in//64, out, 2] strided views below.
|
||||
# layer.weight_scale holds one uint8 scale per 32 input elements.
|
||||
# Pair the scales for the NPU kernel, padding an odd final count
|
||||
# (e.g. K=4304 -> 135 scales -> 136 -> 68 pairs).
|
||||
n_dim, k_dim = layer.weight_scale.data.shape
|
||||
scale = layer.weight_scale.data.reshape(n_dim, k_dim // 2, 2)
|
||||
scale_data = layer.weight_scale.data
|
||||
if k_dim % 2 != 0:
|
||||
scale_data = F.pad(scale_data, (0, 1), mode="constant", value=0)
|
||||
k_dim += 1
|
||||
scale = scale_data.reshape(n_dim, k_dim // 2, 2)
|
||||
layer.weight = Parameter(weight.transpose(0, 1), requires_grad=False)
|
||||
layer.weight_scale_inv = Parameter(
|
||||
scale.transpose(0, 1), requires_grad=False
|
||||
|
||||
@@ -182,9 +182,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if _is_npu:
|
||||
import torch_npu
|
||||
from sgl_kernel_npu.norm.add_rmsnorm_bias import add_gemma_rms_norm
|
||||
|
||||
_NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE = 5120
|
||||
try:
|
||||
from sgl_kernel_npu.norm.gemma_rmsnorm import npu_gemma_rms_norm
|
||||
except ImportError:
|
||||
# sgl-kernel-npu wheels built before the target-specific Gemma provider
|
||||
# landed expose only the torch_npu operator, which is exactly what that
|
||||
# provider binds to on A2/A3 — so those deployments keep working
|
||||
# unchanged. On Ascend 950 the operator is unregistered and the first Gemma
|
||||
# forward fails loudly, same as before this indirection existed.
|
||||
npu_gemma_rms_norm = torch_npu.npu_gemma_rms_norm
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -1237,18 +1244,12 @@ class GemmaRMSNorm(BaseFusedOp):
|
||||
if residual is not None:
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
if x.shape[-1] > _NPU_GEMMA_RMS_NORM_TRITON_MAX_HIDDEN_SIZE:
|
||||
gamma = self.gemma_weight.to(x.dtype)
|
||||
norm_out, _, residual = torch_npu.npu_add_rms_norm(
|
||||
residual, x, gamma, self.variance_epsilon
|
||||
)
|
||||
else:
|
||||
norm_out, residual = add_gemma_rms_norm(
|
||||
x, self.weight, residual, self.variance_epsilon
|
||||
)
|
||||
norm_out, _, residual = torch_npu.npu_add_rms_norm(
|
||||
residual, x, self.gemma_weight.to(x.dtype), self.variance_epsilon
|
||||
)
|
||||
return norm_out, residual
|
||||
|
||||
x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
|
||||
x, _ = npu_gemma_rms_norm(x, self.weight, self.variance_epsilon)
|
||||
return x
|
||||
|
||||
def forward_xpu(
|
||||
@@ -1376,7 +1377,7 @@ class Gemma3RMSNorm(BaseFusedOp):
|
||||
return self.forward_native(x, residual)
|
||||
if residual is not None:
|
||||
return self.forward_native(x, residual)
|
||||
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
|
||||
output, _ = npu_gemma_rms_norm(x, self.weight, self.eps)
|
||||
return output
|
||||
|
||||
def extra_repr(self):
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
Loads weights pre-quantized by msmodelslim (float8_e4m3fn weights,
|
||||
uint8 scales) and runs MXFP8 matmul at inference.
|
||||
|
||||
Following the modelslim-scheme convention (see ModelSlimW8A8Int8), this scheme
|
||||
owns only the hardware-agnostic weight creation; weight post-processing and the
|
||||
forward pass are delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
|
||||
This scheme owns weight creation and alignment of TP shards to checkpoint
|
||||
quantization blocks; weight post-processing and the forward pass are otherwise
|
||||
delegated to an NPUMXFP8LinearMethod kernel (self.kernel). Its
|
||||
process_weights_after_loading detects the pre-quantized float8_e4m3fn weight and
|
||||
takes the offline (transpose-only) branch.
|
||||
"""
|
||||
@@ -13,7 +13,9 @@ takes the offline (transpose-only) branch.
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import (
|
||||
NPUMXFP8LinearMethod,
|
||||
)
|
||||
@@ -60,11 +62,38 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# msmodelslim exports weight_scale as uint8, shape [out, in/32].
|
||||
# msmodelslim exports one uint8 scale per 32 input elements. Keep the
|
||||
# final partial block: Qwen3-VL has visual projections such as K=4304,
|
||||
# whose checkpoint scale shape is [out, ceil(4304/32)] = [out, 135].
|
||||
# NOTE: Named "weight_scale" (not "weight_scale_inv") to match the
|
||||
# checkpoint key exported by msmodelslim; the kernel re-layouts it into
|
||||
# weight_scale_inv during process_weights_after_loading.
|
||||
scale_dim = input_size_per_partition // MXFP8_BLOCK_SIZE
|
||||
scale_dim = (
|
||||
input_size_per_partition + MXFP8_BLOCK_SIZE - 1
|
||||
) // MXFP8_BLOCK_SIZE
|
||||
scale_loader = weight_loader
|
||||
if (
|
||||
input_size_per_partition != input_size
|
||||
and input_size_per_partition % MXFP8_BLOCK_SIZE
|
||||
):
|
||||
tp_rank, _ = get_group_rank_size(layer.tp_group)
|
||||
start = tp_rank * input_size_per_partition
|
||||
left = start % MXFP8_BLOCK_SIZE
|
||||
right = -(left + input_size_per_partition) % MXFP8_BLOCK_SIZE
|
||||
layer.mxfp8_tp_padding = (left, right)
|
||||
scale_start = start // MXFP8_BLOCK_SIZE
|
||||
scale_dim = (left + input_size_per_partition + right) // MXFP8_BLOCK_SIZE
|
||||
|
||||
# A shard can start inside a checkpoint block, so adjacent ranks
|
||||
# may share a scale. Include that block and align both W and X to it.
|
||||
def scale_loader(param, loaded_weight):
|
||||
if not layer.use_presharded_weights:
|
||||
loaded_weight = loaded_weight.narrow(1, scale_start, scale_dim)
|
||||
assert param.shape == loaded_weight.shape, (
|
||||
f"{param.shape=} {loaded_weight.shape=}"
|
||||
)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
(output_size_per_partition, scale_dim),
|
||||
@@ -72,11 +101,13 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
weight_loader=scale_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
if padding := getattr(layer, "mxfp8_tp_padding", None):
|
||||
layer.weight.data = F.pad(layer.weight.data, padding)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
@@ -85,4 +116,11 @@ class ModelSlimMXFP8Scheme(ModelSlimLinearScheme):
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if padding := getattr(layer, "mxfp8_tp_padding", None):
|
||||
if isinstance(x, tuple):
|
||||
raise ValueError(
|
||||
"ModelSlim MXFP8 with unaligned TP shards requires "
|
||||
"unquantized inputs for block alignment."
|
||||
)
|
||||
x = F.pad(x, padding)
|
||||
return self.kernel.apply(layer, x, bias)
|
||||
|
||||
@@ -300,7 +300,8 @@ class MRotaryEmbedding(RotaryEmbedding):
|
||||
assert fused_set_kv_buffer_arg is None, (
|
||||
"fused_set_kv_buffer_arg is not supported for npu implementation"
|
||||
)
|
||||
if query.shape[1] > 4096:
|
||||
# The NPU call below selects ordinary RoPE, which requires 1D positions.
|
||||
if positions.ndim == 2 or query.shape[1] > 4096:
|
||||
return self.forward_native(positions, query, key, fused_set_kv_buffer_arg)
|
||||
rotary_mode = "half" if self.is_neox_style else "interleave"
|
||||
mrope_section = [0, 0, 0]
|
||||
|
||||
@@ -347,6 +347,9 @@ def _select_fused_ar_input_for_linear(hidden_states, linear: nn.Module):
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.norm.split_qkv_rmsnorm_mrope import (
|
||||
triton_split_qkv_rmsnorm_mrope,
|
||||
)
|
||||
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import (
|
||||
split_qkvgate_gemma_rmsnorm_rope,
|
||||
)
|
||||
@@ -1541,6 +1544,31 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
return q, k, v, gate
|
||||
|
||||
def forward_prepare_npu_prefill(self, positions, hidden_states):
|
||||
rotary_emb = self.rotary_emb
|
||||
if not getattr(rotary_emb, "mrope_section", None):
|
||||
return self.forward_prepare_native(positions, hidden_states)
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if positions.ndim == 1:
|
||||
positions = positions.expand(3, -1)
|
||||
cos_sin = rotary_emb.cos_sin_cache[positions].to(qkv)
|
||||
q, k, v, gate = triton_split_qkv_rmsnorm_mrope(
|
||||
qkv=qkv,
|
||||
q_weight=self.q_norm.gemma_weight,
|
||||
k_weight=self.k_norm.gemma_weight,
|
||||
cos_sin=cos_sin,
|
||||
num_q_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
eps=self.q_norm.variance_epsilon,
|
||||
mrope_section=rotary_emb.mrope_section,
|
||||
is_interleaved=rotary_emb.mrope_interleaved,
|
||||
rope_dim=rotary_emb.rotary_dim,
|
||||
has_gate=self.attn_output_gate,
|
||||
)
|
||||
return q, k, v, gate if self.attn_output_gate else None
|
||||
|
||||
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if self.refreshes_rope_cos_sin:
|
||||
@@ -1576,11 +1604,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
if (
|
||||
not _is_npu
|
||||
or forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed()
|
||||
or not self.attn_output_gate
|
||||
):
|
||||
if _is_npu and forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed():
|
||||
return self.forward_prepare_npu_prefill(positions, hidden_states)
|
||||
if not _is_npu or not self.attn_output_gate:
|
||||
return self.forward_prepare_native(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.rotary_embedding import mrope
|
||||
from sglang.srt.layers.rotary_embedding.mrope import (
|
||||
Ernie4_5_VLRotaryEmbedding,
|
||||
MRotaryEmbedding,
|
||||
@@ -104,6 +106,106 @@ class TestMRopeAxisMap(CustomTestCase):
|
||||
)
|
||||
self.assertIsNone(ernie.axis_map)
|
||||
|
||||
def test_npu_dispatch_preserves_multimodal_positions(self):
|
||||
rope = build_mrope([11, 11, 10], 64, interleaved=True)
|
||||
rope.head_size = 256
|
||||
positions = torch.tensor([[0, 1], [2, 3], [4, 5]])
|
||||
query, key = torch.randn(2, 3072), torch.randn(2, 512)
|
||||
expected = rope.forward_native(positions, query, key)
|
||||
with patch.object(mrope, "torch_npu", create=True) as npu:
|
||||
npu.npu_mrope.return_value = (query, key)
|
||||
actual = rope.forward_npu(positions, query, key)
|
||||
npu.npu_mrope.assert_not_called()
|
||||
torch.testing.assert_close(actual, expected)
|
||||
text_positions = positions[0]
|
||||
rope.forward_npu(text_positions, query, key)
|
||||
npu.npu_mrope.assert_called_once_with(
|
||||
text_positions,
|
||||
query,
|
||||
key,
|
||||
rope.cos_sin_cache,
|
||||
256,
|
||||
mrope_section=[0, 0, 0],
|
||||
rotary_mode="half",
|
||||
)
|
||||
|
||||
def test_qwen35_npu_prefill_and_decode_dispatch(self):
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.models import qwen3_5
|
||||
|
||||
layer = SimpleNamespace(
|
||||
attn_output_gate=True,
|
||||
forward_prepare_npu_prefill=Mock(return_value="prefill"),
|
||||
forward_prepare_npu=Mock(return_value="decode"),
|
||||
)
|
||||
with patch.multiple(
|
||||
qwen3_5,
|
||||
_is_npu=True,
|
||||
_is_cuda=False,
|
||||
_is_hip=False,
|
||||
_is_cpu=False,
|
||||
_is_xpu=False,
|
||||
):
|
||||
for mode, expected in (
|
||||
(ForwardMode.EXTEND, "prefill"),
|
||||
(ForwardMode.DECODE, "decode"),
|
||||
):
|
||||
result = qwen3_5.Qwen3_5AttentionDecoderLayer._prepare_qkv_gate(
|
||||
layer, None, None, SimpleNamespace(forward_mode=mode)
|
||||
)
|
||||
self.assertEqual(result, expected)
|
||||
|
||||
def test_qwen35_npu_prefill_mrope_contract(self):
|
||||
from sglang.srt.models import qwen3_5
|
||||
|
||||
# Qwen3.5-27B at TP2: 12 Q/gate heads, 2 K/V heads, partial RoPE.
|
||||
qkv = torch.empty(2, 28 * 256, dtype=torch.bfloat16)
|
||||
rope = build_mrope([11, 11, 10], 64, interleaved=True)
|
||||
layer = SimpleNamespace(
|
||||
qkv_proj=Mock(return_value=(qkv, None)),
|
||||
rotary_emb=rope,
|
||||
num_heads=12,
|
||||
num_kv_heads=2,
|
||||
head_dim=256,
|
||||
attn_output_gate=True,
|
||||
q_norm=SimpleNamespace(
|
||||
gemma_weight=qkv.new_full((256,), 1.25), variance_epsilon=1e-6
|
||||
),
|
||||
k_norm=SimpleNamespace(gemma_weight=qkv.new_full((256,), 0.75)),
|
||||
)
|
||||
multimodal = torch.tensor([[0, 1], [2, 3], [4, 5]])
|
||||
with patch.object(
|
||||
qwen3_5,
|
||||
"triton_split_qkv_rmsnorm_mrope",
|
||||
create=True,
|
||||
return_value=(None, None, None, None),
|
||||
) as fused:
|
||||
for positions in (multimodal, multimodal[0]):
|
||||
qwen3_5.Qwen3_5AttentionDecoderLayer.forward_prepare_npu_prefill(
|
||||
layer, positions, None
|
||||
)
|
||||
args = fused.call_args.kwargs.copy()
|
||||
self.assertIs(args.pop("qkv"), qkv)
|
||||
self.assertIs(args.pop("q_weight"), layer.q_norm.gemma_weight)
|
||||
self.assertIs(args.pop("k_weight"), layer.k_norm.gemma_weight)
|
||||
torch.testing.assert_close(
|
||||
args.pop("cos_sin"),
|
||||
rope.cos_sin_cache[positions.expand(3, -1)].to(qkv),
|
||||
)
|
||||
self.assertEqual(
|
||||
args,
|
||||
dict(
|
||||
num_q_heads=12,
|
||||
num_kv_heads=2,
|
||||
head_size=256,
|
||||
eps=1e-6,
|
||||
mrope_section=[11, 11, 10],
|
||||
is_interleaved=True,
|
||||
rope_dim=64,
|
||||
has_gate=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""CPU regression tests for ModelSlim MXFP8 weight-scale layouts."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization.modelslim.schemes.modelslim_mxfp8 import (
|
||||
ModelSlimMXFP8Scheme,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestModelSlimMXFP8ScaleLayout(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.scheme = ModelSlimMXFP8Scheme()
|
||||
|
||||
def make_layer(self, input_size=4304, tp_size=1, tp_rank=0, presharded=False):
|
||||
layer = torch.nn.Module()
|
||||
layer.tp_group = SimpleNamespace(rank_in_group=tp_rank, world_size=tp_size)
|
||||
layer.use_presharded_weights = presharded
|
||||
self.scheme.create_weights(
|
||||
layer,
|
||||
input_size // tp_size,
|
||||
[2],
|
||||
input_size,
|
||||
2,
|
||||
torch.bfloat16,
|
||||
)
|
||||
return layer
|
||||
|
||||
def test_tp_shards_preserve_checkpoint_blocks(self):
|
||||
k = 4304
|
||||
weight = (torch.arange(2 * k).reshape(2, k) % 7 - 3).to(torch.float8_e4m3fn)
|
||||
scales = (torch.arange(2 * 135).reshape(2, 135) % 7 + 124).to(torch.uint8)
|
||||
reference = (
|
||||
weight.float()
|
||||
* torch.exp2(scales.float() - 127).repeat_interleave(32, dim=1)[:, :k]
|
||||
)
|
||||
x = (torch.arange(k) % 5 - 2).float().unsqueeze(0)
|
||||
|
||||
for tp_size in (1, 2, 4, 8):
|
||||
result = torch.zeros((1, 2))
|
||||
for rank in range(tp_size):
|
||||
with self.subTest(tp_size=tp_size, rank=rank):
|
||||
layer = self.make_layer(tp_size=tp_size, tp_rank=rank)
|
||||
start, end = rank * (k // tp_size), (rank + 1) * (k // tp_size)
|
||||
blocks = torch.arange(start, end) // 32
|
||||
expected_scales = scales[:, blocks.unique_consecutive()]
|
||||
if tp_size == 1:
|
||||
layer.weight_scale.data.copy_(scales)
|
||||
else:
|
||||
layer.weight_scale.weight_loader(layer.weight_scale, scales)
|
||||
torch.testing.assert_close(layer.weight_scale, expected_scales)
|
||||
if tp_size == 4 and rank in (1, 3):
|
||||
self.assertEqual(layer.weight_scale.shape[1], 35)
|
||||
layer.weight.data.copy_(weight[:, start:end])
|
||||
self.scheme.process_weights_after_loading(layer)
|
||||
|
||||
# Undo the kernel's paired scale layout and compare every
|
||||
# real weight element against the global checkpoint blocks.
|
||||
local_scales = layer.weight_scale_inv.transpose(0, 1).reshape(2, -1)
|
||||
local_weight = (
|
||||
layer.weight.t().float()
|
||||
* torch.exp2(local_scales.float() - 127).repeat_interleave(
|
||||
32, dim=1
|
||||
)[:, : layer.weight.shape[0]]
|
||||
)
|
||||
left, right = getattr(layer, "mxfp8_tp_padding", (0, 0))
|
||||
torch.testing.assert_close(
|
||||
local_weight[:, left : left + end - start],
|
||||
reference[:, start:end],
|
||||
)
|
||||
self.assertEqual(torch.count_nonzero(local_weight[:, :left]), 0)
|
||||
if right:
|
||||
self.assertEqual(
|
||||
torch.count_nonzero(local_weight[:, -right:]), 0
|
||||
)
|
||||
padded_x = torch.nn.functional.pad(x[:, start:end], (left, right))
|
||||
result += padded_x @ local_weight.t()
|
||||
torch.testing.assert_close(result, x @ reference.t())
|
||||
|
||||
def test_presharded_scales_are_loaded_without_reslicing(self):
|
||||
layer = self.make_layer(tp_size=4, tp_rank=3, presharded=True)
|
||||
scales = torch.arange(70, dtype=torch.uint8).reshape(2, 35)
|
||||
layer.weight_scale.weight_loader(layer.weight_scale, scales)
|
||||
torch.testing.assert_close(layer.weight_scale, scales)
|
||||
with self.assertRaises(AssertionError):
|
||||
layer.weight_scale.weight_loader(layer.weight_scale, scales[:, :-1])
|
||||
|
||||
def test_unaligned_input_padding_precedes_kernel(self):
|
||||
layer = self.make_layer(tp_size=4, tp_rank=3)
|
||||
self.scheme.kernel = Mock()
|
||||
for shape in ((2, 1076), (2, 3, 1076)):
|
||||
with self.subTest(shape=shape):
|
||||
x = torch.ones(shape, dtype=torch.bfloat16)
|
||||
bias = torch.ones(2)
|
||||
result = self.scheme.apply_weights(layer, x, bias)
|
||||
self.assertIs(result, self.scheme.kernel.apply.return_value)
|
||||
padded = self.scheme.kernel.apply.call_args.args[1]
|
||||
torch.testing.assert_close(padded, torch.nn.functional.pad(x, (28, 16)))
|
||||
self.assertIs(self.scheme.kernel.apply.call_args.args[2], bias)
|
||||
self.scheme.kernel.reset_mock()
|
||||
with self.assertRaisesRegex(ValueError, "unquantized inputs"):
|
||||
self.scheme.apply_weights(layer, (torch.ones(1), torch.ones(1)))
|
||||
self.scheme.kernel.apply.assert_not_called()
|
||||
|
||||
def test_unsharded_column_and_aligned_paths_keep_original_loader(self):
|
||||
for local_k, full_k, output_sizes, output_size in (
|
||||
(4304, 4304, [2], 2), # TP1, partial final block.
|
||||
(4304, 4304, [1], 2), # Column-parallel partition.
|
||||
(96, 192, [2], 2), # Row-parallel, odd but aligned scale count.
|
||||
):
|
||||
with self.subTest(local_k=local_k, output_sizes=output_sizes):
|
||||
layer = torch.nn.Module()
|
||||
loader = Mock()
|
||||
self.scheme.create_weights(
|
||||
layer,
|
||||
local_k,
|
||||
output_sizes,
|
||||
full_k,
|
||||
output_size,
|
||||
torch.bfloat16,
|
||||
weight_loader=loader,
|
||||
)
|
||||
self.assertIs(layer.weight_scale.weight_loader, loader)
|
||||
self.assertIs(layer.weight.weight_loader, loader)
|
||||
self.assertFalse(hasattr(layer, "mxfp8_tp_padding"))
|
||||
self.scheme.kernel = Mock()
|
||||
for x in (torch.ones(2, local_k), (torch.ones(1), torch.ones(1))):
|
||||
self.scheme.apply_weights(layer, x)
|
||||
self.assertIs(self.scheme.kernel.apply.call_args.args[1], x)
|
||||
|
||||
def test_scale_placeholder_rounds_up_partial_block(self):
|
||||
layer = torch.nn.Module()
|
||||
|
||||
self.scheme.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=4304,
|
||||
output_partition_sizes=[2],
|
||||
input_size=4304,
|
||||
output_size=2,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
self.assertEqual(layer.weight_scale.shape, (2, 135))
|
||||
|
||||
def test_post_load_pads_odd_scale_count_for_pair_layout(self):
|
||||
layer = torch.nn.Module()
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(
|
||||
torch.empty((2, 4304), dtype=torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
torch.nn.Parameter(
|
||||
torch.ones((2, 135), dtype=torch.uint8), requires_grad=False
|
||||
),
|
||||
)
|
||||
layer.register_parameter("bias", None)
|
||||
|
||||
self.scheme.process_weights_after_loading(layer)
|
||||
|
||||
self.assertEqual(layer.weight.shape, (4304, 2))
|
||||
self.assertEqual(layer.weight_scale_inv.shape, (68, 2, 2))
|
||||
self.assertTrue(torch.all(layer.weight_scale_inv[-1, :, 1] == 0))
|
||||
self.assertFalse(hasattr(layer, "weight_scale"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,107 @@
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("layer_name", "eps_attr"),
|
||||
[("GemmaRMSNorm", "variance_epsilon"), ("Gemma3RMSNorm", "eps")],
|
||||
)
|
||||
def test_srt_gemma_layers_delegate_plain_npu_path(layer_name, eps_attr):
|
||||
from sglang.srt.layers import layernorm as layernorm_module
|
||||
|
||||
layer = getattr(layernorm_module, layer_name)(4)
|
||||
x = torch.randn(2, 4)
|
||||
unified_op = MagicMock(return_value=(x, "rstd"))
|
||||
|
||||
with patch.object(layernorm_module, "npu_gemma_rms_norm", unified_op, create=True):
|
||||
result = layer.forward_npu(x)
|
||||
|
||||
assert result is x
|
||||
unified_op.assert_called_once_with(x, layer.weight, getattr(layer, eps_attr))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", [256, 5120, 8192])
|
||||
@pytest.mark.parametrize(
|
||||
("weight_dtype", "input_dtype"),
|
||||
[
|
||||
(torch.float16, torch.float16),
|
||||
(torch.bfloat16, torch.bfloat16),
|
||||
(torch.float32, torch.bfloat16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("with_post_residual", [False, True])
|
||||
def test_srt_gemma_residual_reuses_precomputed_weight(
|
||||
hidden_size, weight_dtype, input_dtype, with_post_residual
|
||||
):
|
||||
from sglang.srt.layers import layernorm as layernorm_module
|
||||
|
||||
layer = layernorm_module.GemmaRMSNorm(hidden_size).to(weight_dtype)
|
||||
weight = torch.linspace(-0.5, 0.5, hidden_size, dtype=weight_dtype)
|
||||
layer.weight.weight_loader(layer.weight, weight)
|
||||
x = torch.randn(2, hidden_size, dtype=input_dtype)
|
||||
residual = torch.randn_like(x)
|
||||
post_residual = torch.randn_like(x) if with_post_residual else None
|
||||
output, residual_sum = torch.empty_like(x), torch.empty_like(x)
|
||||
fused_op = MagicMock(return_value=(output, None, residual_sum))
|
||||
|
||||
with patch.object(
|
||||
layernorm_module,
|
||||
"torch_npu",
|
||||
SimpleNamespace(npu_add_rms_norm=fused_op),
|
||||
create=True,
|
||||
):
|
||||
result = layer.forward_npu(x, residual, post_residual)
|
||||
|
||||
assert result[0] is output
|
||||
assert result[1] is residual_sum
|
||||
fused_op.assert_called_once()
|
||||
actual_residual, actual_x, gamma, eps = fused_op.call_args.args
|
||||
assert actual_x is x
|
||||
assert eps == layer.variance_epsilon
|
||||
if with_post_residual:
|
||||
torch.testing.assert_close(actual_residual, residual + post_residual)
|
||||
else:
|
||||
assert actual_residual is residual
|
||||
assert gamma.dtype == input_dtype
|
||||
torch.testing.assert_close(gamma, (1.0 + weight).to(input_dtype))
|
||||
if weight_dtype == input_dtype:
|
||||
assert gamma is layer.gemma_weight
|
||||
|
||||
|
||||
def test_srt_falls_back_to_torch_npu_on_wheels_without_the_provider():
|
||||
"""The srt-layer provider import must stay guarded, with a torch_npu fallback.
|
||||
|
||||
``srt/layers/layernorm.py`` imports the provider at module scope under
|
||||
``if _is_npu``, which CPU CI never executes -- hence a source-shape check.
|
||||
Turning it back into a hard import would break every NPU deployment running
|
||||
an sgl-kernel-npu wheel from before the staged Gemma provider, at
|
||||
``import sglang.srt.layers.layernorm`` time and for all models, not just
|
||||
Gemma ones.
|
||||
"""
|
||||
from sglang.srt.layers import layernorm as layernorm_module
|
||||
|
||||
source = Path(layernorm_module.__file__).read_text(encoding="utf-8")
|
||||
guarded_import = re.search(
|
||||
r"try:\s*\n"
|
||||
r"\s*from sgl_kernel_npu\.norm\.gemma_rmsnorm import npu_gemma_rms_norm\s*\n"
|
||||
r"\s*except ImportError:\s*\n"
|
||||
r"(?:\s*#.*\n)*"
|
||||
r"\s*npu_gemma_rms_norm = torch_npu\.npu_gemma_rms_norm\s*\n",
|
||||
source,
|
||||
)
|
||||
|
||||
assert guarded_import is not None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user