Files
sglang/test/registered/unit/models/test_xllm.py
T

370 lines
13 KiB
Python

# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from types import SimpleNamespace
import pytest
import torch
from sglang.srt.configs.k2_horizon import K2HorizonConfig, XllmConfig
from sglang.srt.models.xllm import (
EntryClass,
K2HorizonForCausalLM,
XllmAttention,
XllmForCausalLM,
XllmGroupRMSNorm,
_normalize_k2_horizon_config,
_validate_mova_config,
_xllm_router_gemm,
_xllm_stacked_params_mapping,
permute_to_hf,
permute_to_xllm,
)
from sglang.srt.runtime_context import get_context
from sglang.srt.utils.hf_transformers.common import _CONFIG_REGISTRY
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=9, suite="base-a-test-cpu")
class _IdentityRotary:
def __call__(self, positions, q, k):
self.positions_shape = tuple(positions.shape)
self.q_shape = tuple(q.shape)
self.k_shape = tuple(k.shape)
return q, k
def test_native_config_and_model_registration():
assert _CONFIG_REGISTRY["xllm"] is XllmConfig
assert _CONFIG_REGISTRY["k2_horizon"] is K2HorizonConfig
assert XllmForCausalLM in EntryClass
assert K2HorizonForCausalLM in EntryClass
def test_dense_horizon_yarn_schema_normalization():
config = K2HorizonConfig.from_dict(
{
"architectures": ["K2HorizonForCausalLM"],
"model_type": "k2_horizon",
"hidden_size": 16,
"num_hidden_layers": 2,
"mlp_only_layers": [0, 1],
"rope_theta": 1_000_000.0,
"max_position_embeddings": 32,
"rope_parameters": {
"rope_type": "yarn",
"factor": 4.0,
"original_max_position_embeddings": 8,
"attention_factor": 1.1,
"beta_fast": 32.0,
"beta_slow": 1.0,
"truncate": True,
},
}
)
_normalize_k2_horizon_config(config)
assert config.num_values == 0
assert config.num_values_per_tok == 0
assert config.num_experts == 0
assert config.num_experts_per_tok == 0
assert config.num_shared_experts == 0
assert config.num_dense_layers == 2
assert config.rope_scaling["rope_type"] == "yarn"
assert config.rope_scaling["original_max_position_embeddings"] == 8
assert config.rope_scaling["attn_factor"] == pytest.approx(
1.1 / (0.1 * math.log(4.0) + 1.0)
)
assert config._sglang_xllm_checkpoint_format == "k2_horizon_hf"
def test_mova_horizon_schema_requires_and_applies_source_router_contract():
config = K2HorizonConfig.from_dict(
{
"architectures": ["K2HorizonForCausalLM"],
"model_type": "k2_horizon",
"hidden_size": 16,
"num_hidden_layers": 4,
"num_experts": 8,
"num_experts_per_tok": 2,
"num_shared_experts": 1,
"mova_num_experts": 4,
"mova_num_experts_per_tok": 2,
"mlp_only_layers": [0],
"attention_gate_func": "softplus",
"rope_parameters": {
"rope_type": "default",
"rope_theta": 10_000_000.0,
},
"xllm_source_router_gemm_partitions": 2,
}
)
_normalize_k2_horizon_config(config)
assert config.num_values == 4
assert config.num_values_per_tok == 2
assert config.num_dense_layers == 1
assert config.apply_attn_gate is True
assert config.attn_gate_func == "softplus"
assert config.rope_theta == 10_000_000.0
def test_mova_horizon_schema_rejects_missing_source_router_contract():
config = K2HorizonConfig.from_dict(
{
"hidden_size": 16,
"num_hidden_layers": 4,
"mova_num_experts": 4,
"mova_num_experts_per_tok": 2,
}
)
with pytest.raises(ValueError, match="source router GEMM provenance"):
_normalize_k2_horizon_config(config)
def test_group_rms_norm_matches_groupwise_reference_and_residual_contract():
norm = XllmGroupRMSNorm(hidden_size=4, n_groups=2, eps=0.0)
with torch.no_grad():
norm.weight.copy_(torch.tensor([1.0, 2.0, 3.0, 4.0]))
hidden = torch.tensor([[3.0, 4.0, 0.0, 5.0]])
residual = torch.ones_like(hidden)
combined = hidden + residual
grouped = combined.reshape(1, 2, 2)
expected = grouped * torch.rsqrt(grouped.square().mean(-1, keepdim=True))
expected = expected.reshape(1, 4) * norm.weight
output, returned_residual = norm(hidden, residual=residual)
torch.testing.assert_close(output, expected)
torch.testing.assert_close(returned_residual, combined)
def test_partial_rope_round_trips_non_rotary_dimensions():
attention = object.__new__(XllmAttention)
torch.nn.Module.__init__(attention)
attention.num_heads = 2
attention.num_kv_heads = 1
attention.head_dim = 8
attention.rope_head_dim = 4
attention.rotary_emb = _IdentityRotary()
positions = torch.arange(3)
q = torch.randn(3, attention.num_heads * attention.head_dim)
k = torch.randn(3, attention.num_kv_heads * attention.head_dim)
q_out, k_out = XllmAttention._apply_partial_rope(attention, positions, q, k)
torch.testing.assert_close(q_out, q)
torch.testing.assert_close(k_out, k)
assert attention.rotary_emb.positions_shape == (3,)
assert attention.rotary_emb.q_shape == (3, 8)
assert attention.rotary_emb.k_shape == (3, 4)
def test_permutation_helpers_use_expected_interleave_order():
hf_value = torch.arange(8, dtype=torch.float32).reshape(1, 1, 8)
xllm_value = torch.tensor([[[0.0, 4.0, 1.0, 5.0, 2.0, 6.0, 3.0, 7.0]]])
torch.testing.assert_close(permute_to_xllm(hf_value), xllm_value)
torch.testing.assert_close(permute_to_hf(xllm_value), hf_value)
def test_mp2_router_gemm_preserves_source_rounding_order():
hidden = torch.tensor([[1.25, -0.75, 0.5, 2.0]], dtype=torch.bfloat16)
weight = torch.tensor(
[[0.25, 1.5, -1.0, 0.75], [2.0, -0.5, 1.25, 0.5]],
dtype=torch.bfloat16,
)
hidden_parts = hidden.chunk(2, dim=-1)
weight_parts = weight.chunk(2, dim=-1)
expected = (
torch.nn.functional.linear(
hidden_parts[0].contiguous(), weight_parts[0].contiguous()
).float()
+ torch.nn.functional.linear(
hidden_parts[1].contiguous(), weight_parts[1].contiguous()
).float()
)
torch.testing.assert_close(_xllm_router_gemm(hidden, weight, 2), expected)
with pytest.raises(ValueError, match="requires BF16"):
_xllm_router_gemm(hidden.float(), weight.float(), 2)
def test_mova_weight_mapping_uses_checkpoint_shaped_attention_projections():
config = SimpleNamespace(num_values=4)
mapping = _xllm_stacked_params_mapping(config)
assert mapping == [
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
(".v_experts.weight", ".v_experts.0.weight", 0),
(".v_experts.weight", ".v_experts.1.weight", 1),
(".v_experts.weight", ".v_experts.2.weight", 2),
(".v_experts.weight", ".v_experts.3.weight", 3),
]
def test_dense_weight_mapping_packs_qkv_and_gate_up():
config = SimpleNamespace(num_values=0)
assert _xllm_stacked_params_mapping(config) == [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
def test_strict_loader_rejects_unknown_checkpoint_weight():
model = object.__new__(XllmForCausalLM)
torch.nn.Module.__init__(model)
model.config = SimpleNamespace(model_type="xllm", tie_word_embeddings=False)
model.model = SimpleNamespace(start_layer=0, end_layer=1)
model.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
model.stacked_params_mapping = []
model.expert_params_mapping = []
with pytest.raises(RuntimeError, match="did not resolve"):
model.load_weights([("unexpected.weight", torch.ones(1))])
def _native_runtime_config(*, enable_two_batch_overlap=False, **overrides):
values = {
"enable_eplb": False,
"init_expert_location": "trivial",
"ep_num_redundant_experts": 0,
"enable_two_batch_overlap": enable_two_batch_overlap,
}
values.update(overrides)
return values
def test_native_xllm_requires_bfloat16(monkeypatch):
config = XllmConfig(num_values=0, num_experts=192)
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.float16)
with (
get_context().override_server_args(**_native_runtime_config()),
pytest.raises(ValueError, match="requires --dtype bfloat16"),
):
_validate_mova_config(config, quant_config=None)
def test_native_xllm_accepts_compressed_tensors_quantization(monkeypatch):
config = XllmConfig(num_values=0, num_experts=0)
quant_config = SimpleNamespace(get_name=lambda: "compressed_tensors")
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with get_context().override_server_args(**_native_runtime_config()):
_validate_mova_config(config, quant_config=quant_config)
def test_native_xllm_rejects_other_quantization(monkeypatch):
config = XllmConfig(num_values=0, num_experts=0)
quant_config = SimpleNamespace(get_name=lambda: "awq")
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with (
get_context().override_server_args(**_native_runtime_config()),
pytest.raises(ValueError, match="supports only compressed-tensors"),
):
_validate_mova_config(config, quant_config=quant_config)
def test_native_xllm_declares_quantized_fused_module_mapping():
assert XllmForCausalLM.packed_modules_mapping == {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
assert K2HorizonForCausalLM.packed_modules_mapping == (
XllmForCausalLM.packed_modules_mapping
)
def test_native_xllm_accepts_bfloat16_without_expert_remapping(monkeypatch):
config = XllmConfig(num_values=0, num_experts=192)
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with get_context().override_server_args(**_native_runtime_config()):
_validate_mova_config(config, quant_config=None)
@pytest.mark.parametrize(
("config_override", "error"),
[
({"query_key_norm": True}, "query/key normalization"),
({"sliding_window": 4096}, "full causal attention only"),
({"use_sliding_window": True}, "full causal attention only"),
({"apply_attn_gate": True}, "gated attention"),
],
ids=["qk-norm", "sliding-window", "use-sliding-window", "attention-gate"],
)
def test_native_dense_xllm_rejects_unimplemented_attention_features(
monkeypatch, config_override, error
):
config = XllmConfig(num_values=0, num_experts=192, **config_override)
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with (
get_context().override_server_args(**_native_runtime_config()),
pytest.raises(ValueError, match=error),
):
_validate_mova_config(config, quant_config=None)
def test_native_xllm_rejects_two_batch_overlap(monkeypatch):
config = XllmConfig(num_values=0, num_experts=192)
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with (
get_context().override_server_args(
**_native_runtime_config(enable_two_batch_overlap=True)
),
pytest.raises(ValueError, match="does not yet support.*two-batch-overlap"),
):
_validate_mova_config(config, quant_config=None)
@pytest.mark.parametrize(
"runtime_override",
[
{"enable_eplb": True},
{"init_expert_location": "random"},
{"ep_num_redundant_experts": 1},
],
ids=["eplb", "initial-placement", "redundant-expert"],
)
def test_native_xllm_rejects_unmapped_expert_modes(monkeypatch, runtime_override):
config = XllmConfig(num_values=0, num_experts=192)
monkeypatch.setattr(torch, "get_default_dtype", lambda: torch.bfloat16)
with (
get_context().override_server_args(
**_native_runtime_config(**runtime_override)
),
pytest.raises(ValueError, match="does not yet support EPLB"),
):
_validate_mova_config(config, quant_config=None)
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))