[AMD] [GLM5] Fuse shared expert into AITER MoE on gfx950 (#41161)
Signed-off-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com> Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com>
This commit is contained in:
co-authored by
Raiden-Makoto
parent
2f88c65289
commit
fa443412c5
@@ -53,6 +53,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
from sglang.srt.layers.quantization.utils import is_layer_skipped
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils.common import PPMissingLayer
|
||||
@@ -1338,6 +1339,8 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
text_config = getattr(hf_config, "text_config", hf_config)
|
||||
if not getattr(text_config, "n_shared_experts", None):
|
||||
return "No shared experts are defined in the config."
|
||||
if getattr(text_config, "n_shared_experts", None) != 1:
|
||||
return "Shared experts fusion requires exactly one shared expert."
|
||||
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
|
||||
first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0)
|
||||
for layer_id in range(first_sparse_layer, text_config.num_hidden_layers):
|
||||
@@ -1349,20 +1352,78 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
"ModelOpt FP4 keeps shared experts unquantized while routed "
|
||||
"experts are quantized."
|
||||
)
|
||||
if not _is_cuda:
|
||||
return "Shared experts fusion currently requires CUDA devices."
|
||||
if _device_sm is not None and _device_sm < 80:
|
||||
return "Shared experts fusion requires SM80 or newer GPUs."
|
||||
if get_parallel().moe_ep_size > 1:
|
||||
return (
|
||||
"Shared experts fusion is not supported together with expert "
|
||||
"parallelism yet."
|
||||
)
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
return (
|
||||
"Shared experts fusion is not supported when Deepep MoE backend "
|
||||
"is enabled."
|
||||
)
|
||||
|
||||
if _is_cuda:
|
||||
if _device_sm is not None and _device_sm < 80:
|
||||
return "Shared experts fusion requires SM80 or newer GPUs."
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
return (
|
||||
"Shared experts fusion is not supported when Deepep MoE backend "
|
||||
"is enabled."
|
||||
)
|
||||
else:
|
||||
if not _use_aiter_gfx95:
|
||||
return "HIP shared experts fusion requires AITER on a gfx950 device."
|
||||
if not get_moe_a2a_backend().is_none():
|
||||
return (
|
||||
"HIP shared experts fusion is not supported when an MoE A2A "
|
||||
"backend is enabled."
|
||||
)
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.tp_size not in (4, 8)
|
||||
or parallel.moe_tp_size != parallel.tp_size
|
||||
):
|
||||
return "HIP shared experts fusion is validated only for TP4 and TP8."
|
||||
if (
|
||||
getattr(text_config, "model_type", None) != "glm5_next_text"
|
||||
or getattr(text_config, "hidden_size", None) != 4096
|
||||
or getattr(text_config, "moe_intermediate_size", None) != 2048
|
||||
or getattr(text_config, "n_routed_experts", None) != 288
|
||||
or getattr(text_config, "num_experts_per_tok", None) != 8
|
||||
):
|
||||
return (
|
||||
"HIP shared experts fusion is validated only for the "
|
||||
"GLM-5.3-Flash E=288/topk=8 expert geometry."
|
||||
)
|
||||
if (
|
||||
getattr(text_config, "hidden_act", None) != "silu"
|
||||
or getattr(text_config, "swiglu_limit", None) != 10.0
|
||||
):
|
||||
return (
|
||||
"HIP shared experts fusion requires the validated "
|
||||
"clamped SiLU G1U1 activation."
|
||||
)
|
||||
if (
|
||||
quant_config is None
|
||||
or quant_config.get_name() != "fp8"
|
||||
or not getattr(quant_config, "is_checkpoint_fp8_serialized", False)
|
||||
or getattr(quant_config, "weight_block_size", None) != [128, 128]
|
||||
or getattr(quant_config, "activation_scheme", None) != "dynamic"
|
||||
):
|
||||
return (
|
||||
"HIP shared experts fusion requires serialized dynamic "
|
||||
"128x128 block-FP8 experts."
|
||||
)
|
||||
ignored_layers = getattr(quant_config, "ignored_layers", [])
|
||||
packed_mapping = getattr(quant_config, "packed_modules_mapping", {})
|
||||
first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0)
|
||||
for layer_id in range(first_sparse_layer, text_config.num_hidden_layers):
|
||||
moe_prefix = f"model.layers.{layer_id}.mlp"
|
||||
if is_layer_skipped(
|
||||
f"{moe_prefix}.experts", ignored_layers, packed_mapping
|
||||
) or is_layer_skipped(
|
||||
f"{moe_prefix}.shared_experts", ignored_layers, packed_mapping
|
||||
):
|
||||
return (
|
||||
"HIP shared experts fusion requires routed and shared "
|
||||
"experts to use the same block-FP8 layout."
|
||||
)
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
"""gfx950 component checks for GLM-5.3 shared-expert AITER fusion."""
|
||||
|
||||
import unittest
|
||||
|
||||
import aiter
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from aiter import dtypes, pertoken_quant
|
||||
from aiter.fused_moe import fused_topk, moe_sorting
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
|
||||
from einops import rearrange
|
||||
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import CustomTestCase, empty_gpu_cache
|
||||
|
||||
# backend-specific: exercises AITER's gfx950 block-FP8 fused-MoE path.
|
||||
register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd-mi35x")
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
is_hip() and is_gfx95_supported(),
|
||||
"GLM-5.3 shared-expert fusion is validated only on gfx950",
|
||||
)
|
||||
class TestGlm53SharedExpertFusion(CustomTestCase):
|
||||
NUM_ROUTED_EXPERTS = 288
|
||||
NUM_FUSED_EXPERTS = 289
|
||||
HIDDEN_SIZE = 4096
|
||||
BLOCK_SIZE = 128
|
||||
ROUTED_TOPK = 8
|
||||
TOKEN_BUCKETS = (1, 8, 128, 2048, 8192, 16384)
|
||||
|
||||
def _quantize_weight(self, weight):
|
||||
experts, out_features, in_features = weight.shape
|
||||
blocks = rearrange(
|
||||
weight.view(
|
||||
experts,
|
||||
out_features // self.BLOCK_SIZE,
|
||||
self.BLOCK_SIZE,
|
||||
in_features // self.BLOCK_SIZE,
|
||||
self.BLOCK_SIZE,
|
||||
),
|
||||
"e nb_n b_n nb_k b_k -> e nb_n nb_k (b_n b_k)",
|
||||
).contiguous()
|
||||
quantized, scale = pertoken_quant(blocks, quant_dtype=dtypes.fp8)
|
||||
quantized = rearrange(
|
||||
quantized.view(
|
||||
experts,
|
||||
out_features // self.BLOCK_SIZE,
|
||||
in_features // self.BLOCK_SIZE,
|
||||
self.BLOCK_SIZE,
|
||||
self.BLOCK_SIZE,
|
||||
),
|
||||
"e nb_n nb_k b_n b_k -> e (nb_n b_n) (nb_k b_k)",
|
||||
).contiguous()
|
||||
return (
|
||||
shuffle_weight(quantized, (16, 16)),
|
||||
scale.view(
|
||||
experts,
|
||||
out_features // self.BLOCK_SIZE,
|
||||
in_features // self.BLOCK_SIZE,
|
||||
).contiguous(),
|
||||
)
|
||||
|
||||
def _make_weights(self, tp_size):
|
||||
intermediate_size = 2048 // tp_size
|
||||
torch.manual_seed(20260921 + tp_size)
|
||||
gate_up = (
|
||||
torch.randn(
|
||||
self.NUM_FUSED_EXPERTS,
|
||||
intermediate_size * 2,
|
||||
self.HIDDEN_SIZE,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
* 0.01
|
||||
)
|
||||
down = (
|
||||
torch.randn(
|
||||
self.NUM_FUSED_EXPERTS,
|
||||
self.HIDDEN_SIZE,
|
||||
intermediate_size,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
* 0.01
|
||||
)
|
||||
gate_up_q, gate_up_scale = self._quantize_weight(gate_up)
|
||||
down_q, down_scale = self._quantize_weight(down)
|
||||
del gate_up, down
|
||||
return gate_up_q, down_q, gate_up_scale, down_scale
|
||||
|
||||
def _fmoe(self, x_q, x_scale, ids, weights, w1, w2, w1_scale, w2_scale):
|
||||
sorted_ids, sorted_weights, sorted_expert_ids, num_valid, output = moe_sorting(
|
||||
ids,
|
||||
weights,
|
||||
w1.shape[0],
|
||||
self.HIDDEN_SIZE,
|
||||
torch.bfloat16,
|
||||
)
|
||||
aiter.fmoe_fp8_blockscale_g1u1(
|
||||
output,
|
||||
x_q,
|
||||
w1,
|
||||
w2,
|
||||
sorted_ids,
|
||||
sorted_weights,
|
||||
sorted_expert_ids,
|
||||
num_valid,
|
||||
ids.shape[1],
|
||||
x_scale.t().contiguous(),
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
"",
|
||||
self.BLOCK_SIZE,
|
||||
self.BLOCK_SIZE,
|
||||
None,
|
||||
)
|
||||
return output
|
||||
|
||||
def _inputs(self, tokens, tp_size):
|
||||
torch.manual_seed(1000 + tokens + tp_size)
|
||||
hidden = (
|
||||
torch.randn(
|
||||
tokens,
|
||||
self.HIDDEN_SIZE,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
* 0.1
|
||||
)
|
||||
scores = torch.randn(
|
||||
tokens,
|
||||
self.NUM_ROUTED_EXPERTS,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
routed_weights, routed_ids = fused_topk(hidden, scores, self.ROUTED_TOPK, True)
|
||||
hidden_q, hidden_scale = pertoken_quant(
|
||||
hidden.view(tokens, self.HIDDEN_SIZE // self.BLOCK_SIZE, self.BLOCK_SIZE),
|
||||
quant_dtype=dtypes.fp8,
|
||||
)
|
||||
return (
|
||||
hidden_q.view(tokens, self.HIDDEN_SIZE),
|
||||
hidden_scale.squeeze(-1),
|
||||
routed_ids,
|
||||
routed_weights,
|
||||
)
|
||||
|
||||
def _separate_and_fused(self, inputs, weights):
|
||||
hidden_q, hidden_scale, routed_ids, routed_weights = inputs
|
||||
w1, w2, w1_scale, w2_scale = weights
|
||||
routed = self._fmoe(
|
||||
hidden_q,
|
||||
hidden_scale,
|
||||
routed_ids,
|
||||
routed_weights,
|
||||
w1[:-1],
|
||||
w2[:-1],
|
||||
w1_scale[:-1],
|
||||
w2_scale[:-1],
|
||||
)
|
||||
gate_up = aiter.gemm_a8w8_blockscale_bpreshuffle(
|
||||
hidden_q,
|
||||
w1[-1],
|
||||
hidden_scale.t().contiguous(),
|
||||
w1_scale[-1],
|
||||
torch.bfloat16,
|
||||
)
|
||||
activated_q, activated_scale = fused_clamp_act_mul(
|
||||
gate_up,
|
||||
dtype_quant=dtypes.fp8,
|
||||
transpose_scale=True,
|
||||
quant_block_size=self.BLOCK_SIZE,
|
||||
)
|
||||
shared = aiter.gemm_a8w8_blockscale_bpreshuffle(
|
||||
activated_q,
|
||||
w2[-1],
|
||||
activated_scale,
|
||||
w2_scale[-1],
|
||||
torch.bfloat16,
|
||||
)
|
||||
separate = routed + shared
|
||||
fused_ids = torch.cat(
|
||||
(
|
||||
routed_ids,
|
||||
torch.full(
|
||||
(routed_ids.shape[0], 1),
|
||||
self.NUM_ROUTED_EXPERTS,
|
||||
device="cuda",
|
||||
dtype=routed_ids.dtype,
|
||||
),
|
||||
),
|
||||
dim=1,
|
||||
)
|
||||
fused_weights = torch.cat(
|
||||
(
|
||||
routed_weights,
|
||||
torch.ones(
|
||||
(routed_weights.shape[0], 1),
|
||||
device="cuda",
|
||||
dtype=routed_weights.dtype,
|
||||
),
|
||||
),
|
||||
dim=1,
|
||||
)
|
||||
fused = self._fmoe(
|
||||
hidden_q,
|
||||
hidden_scale,
|
||||
fused_ids,
|
||||
fused_weights,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
)
|
||||
return separate, fused
|
||||
|
||||
def _assert_matches_separate_path(self, separate, fused):
|
||||
self.assertTrue(torch.isfinite(fused).all())
|
||||
cosine = F.cosine_similarity(
|
||||
fused.float().flatten(),
|
||||
separate.float().flatten(),
|
||||
dim=0,
|
||||
).item()
|
||||
self.assertGreaterEqual(cosine, 0.999)
|
||||
torch.testing.assert_close(fused, separate, rtol=0.05, atol=0.02)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_tp4_tp8_retained_buckets_match_separate_reference(self):
|
||||
"""A wrong appended ID, non-unit shared weight, or incompatible weight
|
||||
layout silently changes model output while every operator still runs."""
|
||||
for tp_size in (4, 8):
|
||||
weights = self._make_weights(tp_size)
|
||||
for tokens in self.TOKEN_BUCKETS:
|
||||
with self.subTest(tp_size=tp_size, tokens=tokens):
|
||||
separate, fused = self._separate_and_fused(
|
||||
self._inputs(tokens, tp_size), weights
|
||||
)
|
||||
self._assert_matches_separate_path(separate, fused)
|
||||
del weights
|
||||
empty_gpu_cache()
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_operator_replays_under_cuda_graph(self):
|
||||
"""The serving path captures decode; eager-only correctness would miss
|
||||
graph-unsafe allocations or stale shared routing tensors."""
|
||||
for tp_size in (4, 8):
|
||||
weights = self._make_weights(tp_size)
|
||||
inputs = self._inputs(8, tp_size)
|
||||
_, eager = self._separate_and_fused(inputs, weights)
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
_, captured = self._separate_and_fused(inputs, weights)
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
torch.testing.assert_close(captured, eager, rtol=0.05, atol=0.02)
|
||||
del graph, captured, eager, inputs, weights
|
||||
empty_gpu_cache()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -18,13 +18,14 @@ from sglang.test.ci.ci_register import register_cpu_ci
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.topk import TopKConfig
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKConfig
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
@@ -158,6 +159,36 @@ class TestFusedSharedExpertScaling(CustomTestCase):
|
||||
expected_shared_id = self.EP_RANK * num_local_experts + num_local_routed
|
||||
self.assertEqual(out_ids[0, -1].item(), expected_shared_id)
|
||||
|
||||
def test_glm53_idle_rank_preserves_routed_topk_contract(self):
|
||||
"""An idle rank must return the pre-append top-8 shape; returning top-9
|
||||
here would make the AITER postprocessor append shared ID 288 twice."""
|
||||
topk = TopK.__new__(TopK)
|
||||
torch.nn.Module.__init__(topk)
|
||||
topk.topk_config = TopKConfig(top_k=9, num_fused_shared_experts=1)
|
||||
topk.enable_waterfill = False
|
||||
topk.waterfill_balancer = None
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
topk_module,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(tp_group=None),
|
||||
),
|
||||
patch.object(topk_module, "is_allocation_symmetric", return_value=False),
|
||||
patch.object(
|
||||
topk_module,
|
||||
"use_symmetric_memory",
|
||||
side_effect=lambda *_args, **_kwargs: nullcontext(),
|
||||
),
|
||||
patch.object(
|
||||
topk_module, "has_per_rank_fused_shared_slots", return_value=False
|
||||
),
|
||||
):
|
||||
output = topk.empty_topk_output(torch.device("cpu"))
|
||||
|
||||
self.assertEqual(output.topk_ids.shape, (0, 8))
|
||||
self.assertEqual(output.topk_weights.shape, (0, 8))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -21,6 +21,7 @@ import unittest.mock
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -58,17 +59,26 @@ class _FusionGateCase(CustomTestCase):
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
|
||||
def _reason(self, model_class, hf_config, quant_config=None, moe_ep_size=1):
|
||||
def _reason(
|
||||
self,
|
||||
model_class,
|
||||
hf_config,
|
||||
quant_config=None,
|
||||
moe_ep_size=1,
|
||||
tp_size=None,
|
||||
):
|
||||
# The gates consult the live EP size; without a group installed the
|
||||
# canonical getter asserts, so every case states a topology.
|
||||
explicit_tp_size = tp_size is not None
|
||||
tp_size = moe_ep_size if tp_size is None else tp_size
|
||||
with get_parallel().override(
|
||||
tp_size=moe_ep_size,
|
||||
attn_tp_size=moe_ep_size,
|
||||
tp_size=tp_size,
|
||||
attn_tp_size=tp_size,
|
||||
attn_dp_size=1,
|
||||
attn_cp_size=1,
|
||||
moe_ep_size=moe_ep_size,
|
||||
moe_dp_size=1,
|
||||
moe_tp_size=1,
|
||||
moe_tp_size=tp_size // moe_ep_size if explicit_tp_size else 1,
|
||||
):
|
||||
return model_class.shared_experts_fusion_disable_reason(
|
||||
hf_config, quant_config
|
||||
@@ -272,6 +282,181 @@ class TestGlmMoeGate(_FusionGateCase):
|
||||
)
|
||||
|
||||
|
||||
class TestGlm5NextGate(_FusionGateCase):
|
||||
def _config(self, **kw):
|
||||
base = dict(
|
||||
model_type="glm5_next_text",
|
||||
hidden_size=4096,
|
||||
moe_intermediate_size=2048,
|
||||
n_routed_experts=288,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=8,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
first_k_dense_replace=3,
|
||||
num_hidden_layers=46,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
def _fp8(self, **kw):
|
||||
base = dict(
|
||||
get_name=lambda: "fp8",
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
weight_block_size=[128, 128],
|
||||
activation_scheme="dynamic",
|
||||
ignored_layers=[],
|
||||
packed_modules_mapping={},
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
def _reason_on_gfx950(self, *, tp_size=4, config=None, quant=None, ep_size=1):
|
||||
import sglang.srt.models.glm5_next as glm5_next
|
||||
|
||||
self._seed()
|
||||
with (
|
||||
unittest.mock.patch.object(glm5_next, "_is_cuda", False),
|
||||
unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", True),
|
||||
):
|
||||
return self._reason(
|
||||
glm5_next.Glm5NextForConditionalGeneration,
|
||||
config or self._config(),
|
||||
quant or self._fp8(),
|
||||
moe_ep_size=ep_size,
|
||||
tp_size=tp_size,
|
||||
)
|
||||
|
||||
def test_exact_tp4_and_tp8_contract_is_admitted(self):
|
||||
for tp_size in (4, 8):
|
||||
with self.subTest(tp_size=tp_size):
|
||||
self.assertIsNone(self._reason_on_gfx950(tp_size=tp_size))
|
||||
|
||||
def test_unvalidated_tp_and_expert_geometry_are_rejected(self):
|
||||
self.assertIn("TP4 and TP8", self._reason_on_gfx950(tp_size=2))
|
||||
self.assertIn(
|
||||
"E=288/topk=8",
|
||||
self._reason_on_gfx950(config=self._config(n_routed_experts=256)),
|
||||
)
|
||||
self.assertIn(
|
||||
"E=288/topk=8",
|
||||
self._reason_on_gfx950(config=self._config(num_experts_per_tok=4)),
|
||||
)
|
||||
|
||||
def test_activation_and_block_fp8_contracts_are_rejected_independently(self):
|
||||
self.assertIn(
|
||||
"clamped SiLU G1U1",
|
||||
self._reason_on_gfx950(config=self._config(swiglu_limit=None)),
|
||||
)
|
||||
self.assertIn(
|
||||
"128x128 block-FP8",
|
||||
self._reason_on_gfx950(quant=self._fp8(weight_block_size=[1, 128])),
|
||||
)
|
||||
self.assertIn(
|
||||
"128x128 block-FP8",
|
||||
self._reason_on_gfx950(quant=_quant("compressed-tensors")),
|
||||
)
|
||||
|
||||
def test_mixed_routed_or_shared_expert_precision_is_rejected(self):
|
||||
for suffix in ("experts", "shared_experts"):
|
||||
with self.subTest(suffix=suffix):
|
||||
quant = self._fp8(ignored_layers=[f"model.layers.3.mlp.{suffix}"])
|
||||
self.assertIn(
|
||||
"same block-FP8 layout",
|
||||
self._reason_on_gfx950(quant=quant),
|
||||
)
|
||||
|
||||
def test_ep_and_a2a_topologies_are_rejected(self):
|
||||
import sglang.srt.models.glm5_next as glm5_next
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend
|
||||
|
||||
self.assertIn(
|
||||
"expert parallelism",
|
||||
self._reason_on_gfx950(tp_size=4, ep_size=2),
|
||||
)
|
||||
with unittest.mock.patch.object(
|
||||
glm5_next, "get_moe_a2a_backend", return_value=MoeA2ABackend.MORI
|
||||
):
|
||||
self.assertIn("A2A backend", self._reason_on_gfx950())
|
||||
|
||||
def test_hip_without_aiter_gfx950_is_rejected(self):
|
||||
import sglang.srt.models.glm5_next as glm5_next
|
||||
|
||||
self._seed()
|
||||
with (
|
||||
unittest.mock.patch.object(glm5_next, "_is_cuda", False),
|
||||
unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", False),
|
||||
):
|
||||
reason = self._reason(
|
||||
glm5_next.Glm5NextForConditionalGeneration,
|
||||
self._config(),
|
||||
self._fp8(),
|
||||
tp_size=4,
|
||||
)
|
||||
self.assertIn("AITER on a gfx950", reason)
|
||||
|
||||
def test_shared_checkpoint_weights_load_into_appended_expert_288(self):
|
||||
"""Enabling fusion must not silently drop the separately named shared
|
||||
tensors; all three projections must reach the appended physical slot."""
|
||||
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
DeepseekV2WeightLoaderMixin,
|
||||
)
|
||||
from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration
|
||||
|
||||
loaded = []
|
||||
|
||||
class Param:
|
||||
def weight_loader(
|
||||
self,
|
||||
_param,
|
||||
loaded_weight,
|
||||
candidate,
|
||||
*,
|
||||
shard_id,
|
||||
expert_id,
|
||||
):
|
||||
loaded.append((loaded_weight, candidate, shard_id, expert_id))
|
||||
|
||||
class Model:
|
||||
config = SimpleNamespace(n_routed_experts=288)
|
||||
num_fused_shared_experts = 1
|
||||
quant_config = None
|
||||
encoder_only = False
|
||||
language_only = True
|
||||
|
||||
def named_parameters(self):
|
||||
return [
|
||||
("model.layers.3.mlp.experts.w13_weight", Param()),
|
||||
("model.layers.3.mlp.experts.w2_weight", Param()),
|
||||
]
|
||||
|
||||
weights = [
|
||||
(
|
||||
f"model.layers.3.mlp.shared_experts.{projection}.weight",
|
||||
torch.full((1,), value),
|
||||
)
|
||||
for value, projection in enumerate(
|
||||
("gate_proj", "down_proj", "up_proj"), start=1
|
||||
)
|
||||
]
|
||||
with unittest.mock.patch.object(
|
||||
DeepseekV2WeightLoaderMixin, "post_load_weights"
|
||||
):
|
||||
Glm5NextForConditionalGeneration.load_weights(Model(), weights)
|
||||
|
||||
self.assertEqual([entry[2] for entry in loaded], ["w1", "w2", "w3"])
|
||||
self.assertEqual([entry[3] for entry in loaded], [288, 288, 288])
|
||||
self.assertEqual(
|
||||
[entry[1] for entry in loaded],
|
||||
[
|
||||
"model.layers.3.mlp.experts.w13_weight",
|
||||
"model.layers.3.mlp.experts.w2_weight",
|
||||
"model.layers.3.mlp.experts.w13_weight",
|
||||
],
|
||||
)
|
||||
self.assertEqual([entry[0].item() for entry in loaded], [1, 2, 3])
|
||||
|
||||
|
||||
class TestMiniMaxGates(_FusionGateCase):
|
||||
def test_a_config_without_shared_experts_cannot_fuse(self):
|
||||
from sglang.srt.models.minimax_m3 import MiniMaxM3SparseForCausalLM
|
||||
|
||||
Reference in New Issue
Block a user