[AMD] [GLM5] Fuse shared expert into AITER MoE on gfx950 (#41161)

Signed-off-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com>
Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com>
This commit is contained in:
Raiden Makoto
2026-09-28 03:40:09 -07:00
committed by GitHub
co-authored by Raiden-Makoto
parent 2f88c65289
commit fa443412c5
4 changed files with 556 additions and 14 deletions
+70 -9
View File
@@ -53,6 +53,7 @@ from sglang.srt.layers.moe.utils import (
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.layers.quantization.utils import is_layer_skipped
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils.common import PPMissingLayer
@@ -1338,6 +1339,8 @@ class Glm5NextForConditionalGeneration(nn.Module):
text_config = getattr(hf_config, "text_config", hf_config)
if not getattr(text_config, "n_shared_experts", None):
return "No shared experts are defined in the config."
if getattr(text_config, "n_shared_experts", None) != 1:
return "Shared experts fusion requires exactly one shared expert."
if quant_config is not None and quant_config.get_name() == "modelopt_fp4":
first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0)
for layer_id in range(first_sparse_layer, text_config.num_hidden_layers):
@@ -1349,20 +1352,78 @@ class Glm5NextForConditionalGeneration(nn.Module):
"ModelOpt FP4 keeps shared experts unquantized while routed "
"experts are quantized."
)
if not _is_cuda:
return "Shared experts fusion currently requires CUDA devices."
if _device_sm is not None and _device_sm < 80:
return "Shared experts fusion requires SM80 or newer GPUs."
if get_parallel().moe_ep_size > 1:
return (
"Shared experts fusion is not supported together with expert "
"parallelism yet."
)
if get_moe_a2a_backend().is_deepep():
return (
"Shared experts fusion is not supported when Deepep MoE backend "
"is enabled."
)
if _is_cuda:
if _device_sm is not None and _device_sm < 80:
return "Shared experts fusion requires SM80 or newer GPUs."
if get_moe_a2a_backend().is_deepep():
return (
"Shared experts fusion is not supported when Deepep MoE backend "
"is enabled."
)
else:
if not _use_aiter_gfx95:
return "HIP shared experts fusion requires AITER on a gfx950 device."
if not get_moe_a2a_backend().is_none():
return (
"HIP shared experts fusion is not supported when an MoE A2A "
"backend is enabled."
)
parallel = get_parallel()
if (
parallel.tp_size not in (4, 8)
or parallel.moe_tp_size != parallel.tp_size
):
return "HIP shared experts fusion is validated only for TP4 and TP8."
if (
getattr(text_config, "model_type", None) != "glm5_next_text"
or getattr(text_config, "hidden_size", None) != 4096
or getattr(text_config, "moe_intermediate_size", None) != 2048
or getattr(text_config, "n_routed_experts", None) != 288
or getattr(text_config, "num_experts_per_tok", None) != 8
):
return (
"HIP shared experts fusion is validated only for the "
"GLM-5.3-Flash E=288/topk=8 expert geometry."
)
if (
getattr(text_config, "hidden_act", None) != "silu"
or getattr(text_config, "swiglu_limit", None) != 10.0
):
return (
"HIP shared experts fusion requires the validated "
"clamped SiLU G1U1 activation."
)
if (
quant_config is None
or quant_config.get_name() != "fp8"
or not getattr(quant_config, "is_checkpoint_fp8_serialized", False)
or getattr(quant_config, "weight_block_size", None) != [128, 128]
or getattr(quant_config, "activation_scheme", None) != "dynamic"
):
return (
"HIP shared experts fusion requires serialized dynamic "
"128x128 block-FP8 experts."
)
ignored_layers = getattr(quant_config, "ignored_layers", [])
packed_mapping = getattr(quant_config, "packed_modules_mapping", {})
first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0)
for layer_id in range(first_sparse_layer, text_config.num_hidden_layers):
moe_prefix = f"model.layers.{layer_id}.mlp"
if is_layer_skipped(
f"{moe_prefix}.experts", ignored_layers, packed_mapping
) or is_layer_skipped(
f"{moe_prefix}.shared_experts", ignored_layers, packed_mapping
):
return (
"HIP shared experts fusion requires routed and shared "
"experts to use the same block-FP8 layout."
)
return None
def determine_num_fused_shared_experts(self):
@@ -0,0 +1,265 @@
"""gfx950 component checks for GLM-5.3 shared-expert AITER fusion."""
import unittest
import aiter
import torch
import torch.nn.functional as F
from aiter import dtypes, pertoken_quant
from aiter.fused_moe import fused_topk, moe_sorting
from aiter.ops.shuffle import shuffle_weight
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from einops import rearrange
from sglang.srt.utils import is_gfx95_supported, is_hip
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import CustomTestCase, empty_gpu_cache
# backend-specific: exercises AITER's gfx950 block-FP8 fused-MoE path.
register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd-mi35x")
@unittest.skipUnless(
is_hip() and is_gfx95_supported(),
"GLM-5.3 shared-expert fusion is validated only on gfx950",
)
class TestGlm53SharedExpertFusion(CustomTestCase):
NUM_ROUTED_EXPERTS = 288
NUM_FUSED_EXPERTS = 289
HIDDEN_SIZE = 4096
BLOCK_SIZE = 128
ROUTED_TOPK = 8
TOKEN_BUCKETS = (1, 8, 128, 2048, 8192, 16384)
def _quantize_weight(self, weight):
experts, out_features, in_features = weight.shape
blocks = rearrange(
weight.view(
experts,
out_features // self.BLOCK_SIZE,
self.BLOCK_SIZE,
in_features // self.BLOCK_SIZE,
self.BLOCK_SIZE,
),
"e nb_n b_n nb_k b_k -> e nb_n nb_k (b_n b_k)",
).contiguous()
quantized, scale = pertoken_quant(blocks, quant_dtype=dtypes.fp8)
quantized = rearrange(
quantized.view(
experts,
out_features // self.BLOCK_SIZE,
in_features // self.BLOCK_SIZE,
self.BLOCK_SIZE,
self.BLOCK_SIZE,
),
"e nb_n nb_k b_n b_k -> e (nb_n b_n) (nb_k b_k)",
).contiguous()
return (
shuffle_weight(quantized, (16, 16)),
scale.view(
experts,
out_features // self.BLOCK_SIZE,
in_features // self.BLOCK_SIZE,
).contiguous(),
)
def _make_weights(self, tp_size):
intermediate_size = 2048 // tp_size
torch.manual_seed(20260921 + tp_size)
gate_up = (
torch.randn(
self.NUM_FUSED_EXPERTS,
intermediate_size * 2,
self.HIDDEN_SIZE,
device="cuda",
dtype=torch.bfloat16,
)
* 0.01
)
down = (
torch.randn(
self.NUM_FUSED_EXPERTS,
self.HIDDEN_SIZE,
intermediate_size,
device="cuda",
dtype=torch.bfloat16,
)
* 0.01
)
gate_up_q, gate_up_scale = self._quantize_weight(gate_up)
down_q, down_scale = self._quantize_weight(down)
del gate_up, down
return gate_up_q, down_q, gate_up_scale, down_scale
def _fmoe(self, x_q, x_scale, ids, weights, w1, w2, w1_scale, w2_scale):
sorted_ids, sorted_weights, sorted_expert_ids, num_valid, output = moe_sorting(
ids,
weights,
w1.shape[0],
self.HIDDEN_SIZE,
torch.bfloat16,
)
aiter.fmoe_fp8_blockscale_g1u1(
output,
x_q,
w1,
w2,
sorted_ids,
sorted_weights,
sorted_expert_ids,
num_valid,
ids.shape[1],
x_scale.t().contiguous(),
w1_scale,
w2_scale,
"",
self.BLOCK_SIZE,
self.BLOCK_SIZE,
None,
)
return output
def _inputs(self, tokens, tp_size):
torch.manual_seed(1000 + tokens + tp_size)
hidden = (
torch.randn(
tokens,
self.HIDDEN_SIZE,
device="cuda",
dtype=torch.bfloat16,
)
* 0.1
)
scores = torch.randn(
tokens,
self.NUM_ROUTED_EXPERTS,
device="cuda",
dtype=torch.bfloat16,
)
routed_weights, routed_ids = fused_topk(hidden, scores, self.ROUTED_TOPK, True)
hidden_q, hidden_scale = pertoken_quant(
hidden.view(tokens, self.HIDDEN_SIZE // self.BLOCK_SIZE, self.BLOCK_SIZE),
quant_dtype=dtypes.fp8,
)
return (
hidden_q.view(tokens, self.HIDDEN_SIZE),
hidden_scale.squeeze(-1),
routed_ids,
routed_weights,
)
def _separate_and_fused(self, inputs, weights):
hidden_q, hidden_scale, routed_ids, routed_weights = inputs
w1, w2, w1_scale, w2_scale = weights
routed = self._fmoe(
hidden_q,
hidden_scale,
routed_ids,
routed_weights,
w1[:-1],
w2[:-1],
w1_scale[:-1],
w2_scale[:-1],
)
gate_up = aiter.gemm_a8w8_blockscale_bpreshuffle(
hidden_q,
w1[-1],
hidden_scale.t().contiguous(),
w1_scale[-1],
torch.bfloat16,
)
activated_q, activated_scale = fused_clamp_act_mul(
gate_up,
dtype_quant=dtypes.fp8,
transpose_scale=True,
quant_block_size=self.BLOCK_SIZE,
)
shared = aiter.gemm_a8w8_blockscale_bpreshuffle(
activated_q,
w2[-1],
activated_scale,
w2_scale[-1],
torch.bfloat16,
)
separate = routed + shared
fused_ids = torch.cat(
(
routed_ids,
torch.full(
(routed_ids.shape[0], 1),
self.NUM_ROUTED_EXPERTS,
device="cuda",
dtype=routed_ids.dtype,
),
),
dim=1,
)
fused_weights = torch.cat(
(
routed_weights,
torch.ones(
(routed_weights.shape[0], 1),
device="cuda",
dtype=routed_weights.dtype,
),
),
dim=1,
)
fused = self._fmoe(
hidden_q,
hidden_scale,
fused_ids,
fused_weights,
w1,
w2,
w1_scale,
w2_scale,
)
return separate, fused
def _assert_matches_separate_path(self, separate, fused):
self.assertTrue(torch.isfinite(fused).all())
cosine = F.cosine_similarity(
fused.float().flatten(),
separate.float().flatten(),
dim=0,
).item()
self.assertGreaterEqual(cosine, 0.999)
torch.testing.assert_close(fused, separate, rtol=0.05, atol=0.02)
@torch.inference_mode()
def test_tp4_tp8_retained_buckets_match_separate_reference(self):
"""A wrong appended ID, non-unit shared weight, or incompatible weight
layout silently changes model output while every operator still runs."""
for tp_size in (4, 8):
weights = self._make_weights(tp_size)
for tokens in self.TOKEN_BUCKETS:
with self.subTest(tp_size=tp_size, tokens=tokens):
separate, fused = self._separate_and_fused(
self._inputs(tokens, tp_size), weights
)
self._assert_matches_separate_path(separate, fused)
del weights
empty_gpu_cache()
@torch.inference_mode()
def test_fused_operator_replays_under_cuda_graph(self):
"""The serving path captures decode; eager-only correctness would miss
graph-unsafe allocations or stale shared routing tensors."""
for tp_size in (4, 8):
weights = self._make_weights(tp_size)
inputs = self._inputs(8, tp_size)
_, eager = self._separate_and_fused(inputs, weights)
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
_, captured = self._separate_and_fused(inputs, weights)
graph.replay()
torch.cuda.synchronize()
torch.testing.assert_close(captured, eager, rtol=0.05, atol=0.02)
del graph, captured, eager, inputs, weights
empty_gpu_cache()
if __name__ == "__main__":
unittest.main()
@@ -18,13 +18,14 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.layers.moe import topk as topk_module
from sglang.srt.layers.moe.topk import TopKConfig
from sglang.srt.layers.moe.topk import TopK, TopKConfig
from sglang.test.test_utils import CustomTestCase
@@ -158,6 +159,36 @@ class TestFusedSharedExpertScaling(CustomTestCase):
expected_shared_id = self.EP_RANK * num_local_experts + num_local_routed
self.assertEqual(out_ids[0, -1].item(), expected_shared_id)
def test_glm53_idle_rank_preserves_routed_topk_contract(self):
"""An idle rank must return the pre-append top-8 shape; returning top-9
here would make the AITER postprocessor append shared ID 288 twice."""
topk = TopK.__new__(TopK)
torch.nn.Module.__init__(topk)
topk.topk_config = TopKConfig(top_k=9, num_fused_shared_experts=1)
topk.enable_waterfill = False
topk.waterfill_balancer = None
with (
patch.object(
topk_module,
"get_parallel",
return_value=SimpleNamespace(tp_group=None),
),
patch.object(topk_module, "is_allocation_symmetric", return_value=False),
patch.object(
topk_module,
"use_symmetric_memory",
side_effect=lambda *_args, **_kwargs: nullcontext(),
),
patch.object(
topk_module, "has_per_rank_fused_shared_slots", return_value=False
),
):
output = topk.empty_topk_output(torch.device("cpu"))
self.assertEqual(output.topk_ids.shape, (0, 8))
self.assertEqual(output.topk_weights.shape, (0, 8))
if __name__ == "__main__":
unittest.main()
@@ -21,6 +21,7 @@ import unittest.mock
from types import ModuleType, SimpleNamespace
import pytest
import torch
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
@@ -58,17 +59,26 @@ class _FusionGateCase(CustomTestCase):
override.install()
self.addCleanup(override.restore)
def _reason(self, model_class, hf_config, quant_config=None, moe_ep_size=1):
def _reason(
self,
model_class,
hf_config,
quant_config=None,
moe_ep_size=1,
tp_size=None,
):
# The gates consult the live EP size; without a group installed the
# canonical getter asserts, so every case states a topology.
explicit_tp_size = tp_size is not None
tp_size = moe_ep_size if tp_size is None else tp_size
with get_parallel().override(
tp_size=moe_ep_size,
attn_tp_size=moe_ep_size,
tp_size=tp_size,
attn_tp_size=tp_size,
attn_dp_size=1,
attn_cp_size=1,
moe_ep_size=moe_ep_size,
moe_dp_size=1,
moe_tp_size=1,
moe_tp_size=tp_size // moe_ep_size if explicit_tp_size else 1,
):
return model_class.shared_experts_fusion_disable_reason(
hf_config, quant_config
@@ -272,6 +282,181 @@ class TestGlmMoeGate(_FusionGateCase):
)
class TestGlm5NextGate(_FusionGateCase):
def _config(self, **kw):
base = dict(
model_type="glm5_next_text",
hidden_size=4096,
moe_intermediate_size=2048,
n_routed_experts=288,
n_shared_experts=1,
num_experts_per_tok=8,
hidden_act="silu",
swiglu_limit=10.0,
first_k_dense_replace=3,
num_hidden_layers=46,
)
base.update(kw)
return SimpleNamespace(**base)
def _fp8(self, **kw):
base = dict(
get_name=lambda: "fp8",
is_checkpoint_fp8_serialized=True,
weight_block_size=[128, 128],
activation_scheme="dynamic",
ignored_layers=[],
packed_modules_mapping={},
)
base.update(kw)
return SimpleNamespace(**base)
def _reason_on_gfx950(self, *, tp_size=4, config=None, quant=None, ep_size=1):
import sglang.srt.models.glm5_next as glm5_next
self._seed()
with (
unittest.mock.patch.object(glm5_next, "_is_cuda", False),
unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", True),
):
return self._reason(
glm5_next.Glm5NextForConditionalGeneration,
config or self._config(),
quant or self._fp8(),
moe_ep_size=ep_size,
tp_size=tp_size,
)
def test_exact_tp4_and_tp8_contract_is_admitted(self):
for tp_size in (4, 8):
with self.subTest(tp_size=tp_size):
self.assertIsNone(self._reason_on_gfx950(tp_size=tp_size))
def test_unvalidated_tp_and_expert_geometry_are_rejected(self):
self.assertIn("TP4 and TP8", self._reason_on_gfx950(tp_size=2))
self.assertIn(
"E=288/topk=8",
self._reason_on_gfx950(config=self._config(n_routed_experts=256)),
)
self.assertIn(
"E=288/topk=8",
self._reason_on_gfx950(config=self._config(num_experts_per_tok=4)),
)
def test_activation_and_block_fp8_contracts_are_rejected_independently(self):
self.assertIn(
"clamped SiLU G1U1",
self._reason_on_gfx950(config=self._config(swiglu_limit=None)),
)
self.assertIn(
"128x128 block-FP8",
self._reason_on_gfx950(quant=self._fp8(weight_block_size=[1, 128])),
)
self.assertIn(
"128x128 block-FP8",
self._reason_on_gfx950(quant=_quant("compressed-tensors")),
)
def test_mixed_routed_or_shared_expert_precision_is_rejected(self):
for suffix in ("experts", "shared_experts"):
with self.subTest(suffix=suffix):
quant = self._fp8(ignored_layers=[f"model.layers.3.mlp.{suffix}"])
self.assertIn(
"same block-FP8 layout",
self._reason_on_gfx950(quant=quant),
)
def test_ep_and_a2a_topologies_are_rejected(self):
import sglang.srt.models.glm5_next as glm5_next
from sglang.srt.layers.moe.utils import MoeA2ABackend
self.assertIn(
"expert parallelism",
self._reason_on_gfx950(tp_size=4, ep_size=2),
)
with unittest.mock.patch.object(
glm5_next, "get_moe_a2a_backend", return_value=MoeA2ABackend.MORI
):
self.assertIn("A2A backend", self._reason_on_gfx950())
def test_hip_without_aiter_gfx950_is_rejected(self):
import sglang.srt.models.glm5_next as glm5_next
self._seed()
with (
unittest.mock.patch.object(glm5_next, "_is_cuda", False),
unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", False),
):
reason = self._reason(
glm5_next.Glm5NextForConditionalGeneration,
self._config(),
self._fp8(),
tp_size=4,
)
self.assertIn("AITER on a gfx950", reason)
def test_shared_checkpoint_weights_load_into_appended_expert_288(self):
"""Enabling fusion must not silently drop the separately named shared
tensors; all three projections must reach the appended physical slot."""
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
DeepseekV2WeightLoaderMixin,
)
from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration
loaded = []
class Param:
def weight_loader(
self,
_param,
loaded_weight,
candidate,
*,
shard_id,
expert_id,
):
loaded.append((loaded_weight, candidate, shard_id, expert_id))
class Model:
config = SimpleNamespace(n_routed_experts=288)
num_fused_shared_experts = 1
quant_config = None
encoder_only = False
language_only = True
def named_parameters(self):
return [
("model.layers.3.mlp.experts.w13_weight", Param()),
("model.layers.3.mlp.experts.w2_weight", Param()),
]
weights = [
(
f"model.layers.3.mlp.shared_experts.{projection}.weight",
torch.full((1,), value),
)
for value, projection in enumerate(
("gate_proj", "down_proj", "up_proj"), start=1
)
]
with unittest.mock.patch.object(
DeepseekV2WeightLoaderMixin, "post_load_weights"
):
Glm5NextForConditionalGeneration.load_weights(Model(), weights)
self.assertEqual([entry[2] for entry in loaded], ["w1", "w2", "w3"])
self.assertEqual([entry[3] for entry in loaded], [288, 288, 288])
self.assertEqual(
[entry[1] for entry in loaded],
[
"model.layers.3.mlp.experts.w13_weight",
"model.layers.3.mlp.experts.w2_weight",
"model.layers.3.mlp.experts.w13_weight",
],
)
self.assertEqual([entry[0].item() for entry in loaded], [1, 2, 3])
class TestMiniMaxGates(_FusionGateCase):
def test_a_config_without_shared_experts_cannot_fuse(self):
from sglang.srt.models.minimax_m3 import MiniMaxM3SparseForCausalLM