diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 81ce2b67b7..455392dadc 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -53,6 +53,7 @@ from sglang.srt.layers.moe.utils import ( ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import is_layer_skipped from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils.common import PPMissingLayer @@ -1338,6 +1339,8 @@ class Glm5NextForConditionalGeneration(nn.Module): text_config = getattr(hf_config, "text_config", hf_config) if not getattr(text_config, "n_shared_experts", None): return "No shared experts are defined in the config." + if getattr(text_config, "n_shared_experts", None) != 1: + return "Shared experts fusion requires exactly one shared expert." if quant_config is not None and quant_config.get_name() == "modelopt_fp4": first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0) for layer_id in range(first_sparse_layer, text_config.num_hidden_layers): @@ -1349,20 +1352,78 @@ class Glm5NextForConditionalGeneration(nn.Module): "ModelOpt FP4 keeps shared experts unquantized while routed " "experts are quantized." ) - if not _is_cuda: - return "Shared experts fusion currently requires CUDA devices." - if _device_sm is not None and _device_sm < 80: - return "Shared experts fusion requires SM80 or newer GPUs." if get_parallel().moe_ep_size > 1: return ( "Shared experts fusion is not supported together with expert " "parallelism yet." ) - if get_moe_a2a_backend().is_deepep(): - return ( - "Shared experts fusion is not supported when Deepep MoE backend " - "is enabled." - ) + + if _is_cuda: + if _device_sm is not None and _device_sm < 80: + return "Shared experts fusion requires SM80 or newer GPUs." + if get_moe_a2a_backend().is_deepep(): + return ( + "Shared experts fusion is not supported when Deepep MoE backend " + "is enabled." + ) + else: + if not _use_aiter_gfx95: + return "HIP shared experts fusion requires AITER on a gfx950 device." + if not get_moe_a2a_backend().is_none(): + return ( + "HIP shared experts fusion is not supported when an MoE A2A " + "backend is enabled." + ) + parallel = get_parallel() + if ( + parallel.tp_size not in (4, 8) + or parallel.moe_tp_size != parallel.tp_size + ): + return "HIP shared experts fusion is validated only for TP4 and TP8." + if ( + getattr(text_config, "model_type", None) != "glm5_next_text" + or getattr(text_config, "hidden_size", None) != 4096 + or getattr(text_config, "moe_intermediate_size", None) != 2048 + or getattr(text_config, "n_routed_experts", None) != 288 + or getattr(text_config, "num_experts_per_tok", None) != 8 + ): + return ( + "HIP shared experts fusion is validated only for the " + "GLM-5.3-Flash E=288/topk=8 expert geometry." + ) + if ( + getattr(text_config, "hidden_act", None) != "silu" + or getattr(text_config, "swiglu_limit", None) != 10.0 + ): + return ( + "HIP shared experts fusion requires the validated " + "clamped SiLU G1U1 activation." + ) + if ( + quant_config is None + or quant_config.get_name() != "fp8" + or not getattr(quant_config, "is_checkpoint_fp8_serialized", False) + or getattr(quant_config, "weight_block_size", None) != [128, 128] + or getattr(quant_config, "activation_scheme", None) != "dynamic" + ): + return ( + "HIP shared experts fusion requires serialized dynamic " + "128x128 block-FP8 experts." + ) + ignored_layers = getattr(quant_config, "ignored_layers", []) + packed_mapping = getattr(quant_config, "packed_modules_mapping", {}) + first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0) + for layer_id in range(first_sparse_layer, text_config.num_hidden_layers): + moe_prefix = f"model.layers.{layer_id}.mlp" + if is_layer_skipped( + f"{moe_prefix}.experts", ignored_layers, packed_mapping + ) or is_layer_skipped( + f"{moe_prefix}.shared_experts", ignored_layers, packed_mapping + ): + return ( + "HIP shared experts fusion requires routed and shared " + "experts to use the same block-FP8 layout." + ) return None def determine_num_fused_shared_experts(self): diff --git a/test/registered/kernels/ops/moe/test_glm53_shared_expert_fusion.py b/test/registered/kernels/ops/moe/test_glm53_shared_expert_fusion.py new file mode 100644 index 0000000000..10b526ecf2 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_glm53_shared_expert_fusion.py @@ -0,0 +1,265 @@ +"""gfx950 component checks for GLM-5.3 shared-expert AITER fusion.""" + +import unittest + +import aiter +import torch +import torch.nn.functional as F +from aiter import dtypes, pertoken_quant +from aiter.fused_moe import fused_topk, moe_sorting +from aiter.ops.shuffle import shuffle_weight +from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul +from einops import rearrange + +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase, empty_gpu_cache + +# backend-specific: exercises AITER's gfx950 block-FP8 fused-MoE path. +register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd-mi35x") + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported(), + "GLM-5.3 shared-expert fusion is validated only on gfx950", +) +class TestGlm53SharedExpertFusion(CustomTestCase): + NUM_ROUTED_EXPERTS = 288 + NUM_FUSED_EXPERTS = 289 + HIDDEN_SIZE = 4096 + BLOCK_SIZE = 128 + ROUTED_TOPK = 8 + TOKEN_BUCKETS = (1, 8, 128, 2048, 8192, 16384) + + def _quantize_weight(self, weight): + experts, out_features, in_features = weight.shape + blocks = rearrange( + weight.view( + experts, + out_features // self.BLOCK_SIZE, + self.BLOCK_SIZE, + in_features // self.BLOCK_SIZE, + self.BLOCK_SIZE, + ), + "e nb_n b_n nb_k b_k -> e nb_n nb_k (b_n b_k)", + ).contiguous() + quantized, scale = pertoken_quant(blocks, quant_dtype=dtypes.fp8) + quantized = rearrange( + quantized.view( + experts, + out_features // self.BLOCK_SIZE, + in_features // self.BLOCK_SIZE, + self.BLOCK_SIZE, + self.BLOCK_SIZE, + ), + "e nb_n nb_k b_n b_k -> e (nb_n b_n) (nb_k b_k)", + ).contiguous() + return ( + shuffle_weight(quantized, (16, 16)), + scale.view( + experts, + out_features // self.BLOCK_SIZE, + in_features // self.BLOCK_SIZE, + ).contiguous(), + ) + + def _make_weights(self, tp_size): + intermediate_size = 2048 // tp_size + torch.manual_seed(20260921 + tp_size) + gate_up = ( + torch.randn( + self.NUM_FUSED_EXPERTS, + intermediate_size * 2, + self.HIDDEN_SIZE, + device="cuda", + dtype=torch.bfloat16, + ) + * 0.01 + ) + down = ( + torch.randn( + self.NUM_FUSED_EXPERTS, + self.HIDDEN_SIZE, + intermediate_size, + device="cuda", + dtype=torch.bfloat16, + ) + * 0.01 + ) + gate_up_q, gate_up_scale = self._quantize_weight(gate_up) + down_q, down_scale = self._quantize_weight(down) + del gate_up, down + return gate_up_q, down_q, gate_up_scale, down_scale + + def _fmoe(self, x_q, x_scale, ids, weights, w1, w2, w1_scale, w2_scale): + sorted_ids, sorted_weights, sorted_expert_ids, num_valid, output = moe_sorting( + ids, + weights, + w1.shape[0], + self.HIDDEN_SIZE, + torch.bfloat16, + ) + aiter.fmoe_fp8_blockscale_g1u1( + output, + x_q, + w1, + w2, + sorted_ids, + sorted_weights, + sorted_expert_ids, + num_valid, + ids.shape[1], + x_scale.t().contiguous(), + w1_scale, + w2_scale, + "", + self.BLOCK_SIZE, + self.BLOCK_SIZE, + None, + ) + return output + + def _inputs(self, tokens, tp_size): + torch.manual_seed(1000 + tokens + tp_size) + hidden = ( + torch.randn( + tokens, + self.HIDDEN_SIZE, + device="cuda", + dtype=torch.bfloat16, + ) + * 0.1 + ) + scores = torch.randn( + tokens, + self.NUM_ROUTED_EXPERTS, + device="cuda", + dtype=torch.bfloat16, + ) + routed_weights, routed_ids = fused_topk(hidden, scores, self.ROUTED_TOPK, True) + hidden_q, hidden_scale = pertoken_quant( + hidden.view(tokens, self.HIDDEN_SIZE // self.BLOCK_SIZE, self.BLOCK_SIZE), + quant_dtype=dtypes.fp8, + ) + return ( + hidden_q.view(tokens, self.HIDDEN_SIZE), + hidden_scale.squeeze(-1), + routed_ids, + routed_weights, + ) + + def _separate_and_fused(self, inputs, weights): + hidden_q, hidden_scale, routed_ids, routed_weights = inputs + w1, w2, w1_scale, w2_scale = weights + routed = self._fmoe( + hidden_q, + hidden_scale, + routed_ids, + routed_weights, + w1[:-1], + w2[:-1], + w1_scale[:-1], + w2_scale[:-1], + ) + gate_up = aiter.gemm_a8w8_blockscale_bpreshuffle( + hidden_q, + w1[-1], + hidden_scale.t().contiguous(), + w1_scale[-1], + torch.bfloat16, + ) + activated_q, activated_scale = fused_clamp_act_mul( + gate_up, + dtype_quant=dtypes.fp8, + transpose_scale=True, + quant_block_size=self.BLOCK_SIZE, + ) + shared = aiter.gemm_a8w8_blockscale_bpreshuffle( + activated_q, + w2[-1], + activated_scale, + w2_scale[-1], + torch.bfloat16, + ) + separate = routed + shared + fused_ids = torch.cat( + ( + routed_ids, + torch.full( + (routed_ids.shape[0], 1), + self.NUM_ROUTED_EXPERTS, + device="cuda", + dtype=routed_ids.dtype, + ), + ), + dim=1, + ) + fused_weights = torch.cat( + ( + routed_weights, + torch.ones( + (routed_weights.shape[0], 1), + device="cuda", + dtype=routed_weights.dtype, + ), + ), + dim=1, + ) + fused = self._fmoe( + hidden_q, + hidden_scale, + fused_ids, + fused_weights, + w1, + w2, + w1_scale, + w2_scale, + ) + return separate, fused + + def _assert_matches_separate_path(self, separate, fused): + self.assertTrue(torch.isfinite(fused).all()) + cosine = F.cosine_similarity( + fused.float().flatten(), + separate.float().flatten(), + dim=0, + ).item() + self.assertGreaterEqual(cosine, 0.999) + torch.testing.assert_close(fused, separate, rtol=0.05, atol=0.02) + + @torch.inference_mode() + def test_tp4_tp8_retained_buckets_match_separate_reference(self): + """A wrong appended ID, non-unit shared weight, or incompatible weight + layout silently changes model output while every operator still runs.""" + for tp_size in (4, 8): + weights = self._make_weights(tp_size) + for tokens in self.TOKEN_BUCKETS: + with self.subTest(tp_size=tp_size, tokens=tokens): + separate, fused = self._separate_and_fused( + self._inputs(tokens, tp_size), weights + ) + self._assert_matches_separate_path(separate, fused) + del weights + empty_gpu_cache() + + @torch.inference_mode() + def test_fused_operator_replays_under_cuda_graph(self): + """The serving path captures decode; eager-only correctness would miss + graph-unsafe allocations or stale shared routing tensors.""" + for tp_size in (4, 8): + weights = self._make_weights(tp_size) + inputs = self._inputs(8, tp_size) + _, eager = self._separate_and_fused(inputs, weights) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _, captured = self._separate_and_fused(inputs, weights) + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close(captured, eager, rtol=0.05, atol=0.02) + del graph, captured, eager, inputs, weights + empty_gpu_cache() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index 4f3945e525..3faad1deb1 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -18,13 +18,14 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") import unittest +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import patch import torch from sglang.srt.layers.moe import topk as topk_module -from sglang.srt.layers.moe.topk import TopKConfig +from sglang.srt.layers.moe.topk import TopK, TopKConfig from sglang.test.test_utils import CustomTestCase @@ -158,6 +159,36 @@ class TestFusedSharedExpertScaling(CustomTestCase): expected_shared_id = self.EP_RANK * num_local_experts + num_local_routed self.assertEqual(out_ids[0, -1].item(), expected_shared_id) + def test_glm53_idle_rank_preserves_routed_topk_contract(self): + """An idle rank must return the pre-append top-8 shape; returning top-9 + here would make the AITER postprocessor append shared ID 288 twice.""" + topk = TopK.__new__(TopK) + torch.nn.Module.__init__(topk) + topk.topk_config = TopKConfig(top_k=9, num_fused_shared_experts=1) + topk.enable_waterfill = False + topk.waterfill_balancer = None + + with ( + patch.object( + topk_module, + "get_parallel", + return_value=SimpleNamespace(tp_group=None), + ), + patch.object(topk_module, "is_allocation_symmetric", return_value=False), + patch.object( + topk_module, + "use_symmetric_memory", + side_effect=lambda *_args, **_kwargs: nullcontext(), + ), + patch.object( + topk_module, "has_per_rank_fused_shared_slots", return_value=False + ), + ): + output = topk.empty_topk_output(torch.device("cpu")) + + self.assertEqual(output.topk_ids.shape, (0, 8)) + self.assertEqual(output.topk_weights.shape, (0, 8)) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_shared_experts_fusion_gates.py b/test/registered/unit/models/test_shared_experts_fusion_gates.py index 96f6dbe4bb..273136c0fc 100644 --- a/test/registered/unit/models/test_shared_experts_fusion_gates.py +++ b/test/registered/unit/models/test_shared_experts_fusion_gates.py @@ -21,6 +21,7 @@ import unittest.mock from types import ModuleType, SimpleNamespace import pytest +import torch from sglang.srt.runtime_context import get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci @@ -58,17 +59,26 @@ class _FusionGateCase(CustomTestCase): override.install() self.addCleanup(override.restore) - def _reason(self, model_class, hf_config, quant_config=None, moe_ep_size=1): + def _reason( + self, + model_class, + hf_config, + quant_config=None, + moe_ep_size=1, + tp_size=None, + ): # The gates consult the live EP size; without a group installed the # canonical getter asserts, so every case states a topology. + explicit_tp_size = tp_size is not None + tp_size = moe_ep_size if tp_size is None else tp_size with get_parallel().override( - tp_size=moe_ep_size, - attn_tp_size=moe_ep_size, + tp_size=tp_size, + attn_tp_size=tp_size, attn_dp_size=1, attn_cp_size=1, moe_ep_size=moe_ep_size, moe_dp_size=1, - moe_tp_size=1, + moe_tp_size=tp_size // moe_ep_size if explicit_tp_size else 1, ): return model_class.shared_experts_fusion_disable_reason( hf_config, quant_config @@ -272,6 +282,181 @@ class TestGlmMoeGate(_FusionGateCase): ) +class TestGlm5NextGate(_FusionGateCase): + def _config(self, **kw): + base = dict( + model_type="glm5_next_text", + hidden_size=4096, + moe_intermediate_size=2048, + n_routed_experts=288, + n_shared_experts=1, + num_experts_per_tok=8, + hidden_act="silu", + swiglu_limit=10.0, + first_k_dense_replace=3, + num_hidden_layers=46, + ) + base.update(kw) + return SimpleNamespace(**base) + + def _fp8(self, **kw): + base = dict( + get_name=lambda: "fp8", + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + activation_scheme="dynamic", + ignored_layers=[], + packed_modules_mapping={}, + ) + base.update(kw) + return SimpleNamespace(**base) + + def _reason_on_gfx950(self, *, tp_size=4, config=None, quant=None, ep_size=1): + import sglang.srt.models.glm5_next as glm5_next + + self._seed() + with ( + unittest.mock.patch.object(glm5_next, "_is_cuda", False), + unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", True), + ): + return self._reason( + glm5_next.Glm5NextForConditionalGeneration, + config or self._config(), + quant or self._fp8(), + moe_ep_size=ep_size, + tp_size=tp_size, + ) + + def test_exact_tp4_and_tp8_contract_is_admitted(self): + for tp_size in (4, 8): + with self.subTest(tp_size=tp_size): + self.assertIsNone(self._reason_on_gfx950(tp_size=tp_size)) + + def test_unvalidated_tp_and_expert_geometry_are_rejected(self): + self.assertIn("TP4 and TP8", self._reason_on_gfx950(tp_size=2)) + self.assertIn( + "E=288/topk=8", + self._reason_on_gfx950(config=self._config(n_routed_experts=256)), + ) + self.assertIn( + "E=288/topk=8", + self._reason_on_gfx950(config=self._config(num_experts_per_tok=4)), + ) + + def test_activation_and_block_fp8_contracts_are_rejected_independently(self): + self.assertIn( + "clamped SiLU G1U1", + self._reason_on_gfx950(config=self._config(swiglu_limit=None)), + ) + self.assertIn( + "128x128 block-FP8", + self._reason_on_gfx950(quant=self._fp8(weight_block_size=[1, 128])), + ) + self.assertIn( + "128x128 block-FP8", + self._reason_on_gfx950(quant=_quant("compressed-tensors")), + ) + + def test_mixed_routed_or_shared_expert_precision_is_rejected(self): + for suffix in ("experts", "shared_experts"): + with self.subTest(suffix=suffix): + quant = self._fp8(ignored_layers=[f"model.layers.3.mlp.{suffix}"]) + self.assertIn( + "same block-FP8 layout", + self._reason_on_gfx950(quant=quant), + ) + + def test_ep_and_a2a_topologies_are_rejected(self): + import sglang.srt.models.glm5_next as glm5_next + from sglang.srt.layers.moe.utils import MoeA2ABackend + + self.assertIn( + "expert parallelism", + self._reason_on_gfx950(tp_size=4, ep_size=2), + ) + with unittest.mock.patch.object( + glm5_next, "get_moe_a2a_backend", return_value=MoeA2ABackend.MORI + ): + self.assertIn("A2A backend", self._reason_on_gfx950()) + + def test_hip_without_aiter_gfx950_is_rejected(self): + import sglang.srt.models.glm5_next as glm5_next + + self._seed() + with ( + unittest.mock.patch.object(glm5_next, "_is_cuda", False), + unittest.mock.patch.object(glm5_next, "_use_aiter_gfx95", False), + ): + reason = self._reason( + glm5_next.Glm5NextForConditionalGeneration, + self._config(), + self._fp8(), + tp_size=4, + ) + self.assertIn("AITER on a gfx950", reason) + + def test_shared_checkpoint_weights_load_into_appended_expert_288(self): + """Enabling fusion must not silently drop the separately named shared + tensors; all three projections must reach the appended physical slot.""" + from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( + DeepseekV2WeightLoaderMixin, + ) + from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration + + loaded = [] + + class Param: + def weight_loader( + self, + _param, + loaded_weight, + candidate, + *, + shard_id, + expert_id, + ): + loaded.append((loaded_weight, candidate, shard_id, expert_id)) + + class Model: + config = SimpleNamespace(n_routed_experts=288) + num_fused_shared_experts = 1 + quant_config = None + encoder_only = False + language_only = True + + def named_parameters(self): + return [ + ("model.layers.3.mlp.experts.w13_weight", Param()), + ("model.layers.3.mlp.experts.w2_weight", Param()), + ] + + weights = [ + ( + f"model.layers.3.mlp.shared_experts.{projection}.weight", + torch.full((1,), value), + ) + for value, projection in enumerate( + ("gate_proj", "down_proj", "up_proj"), start=1 + ) + ] + with unittest.mock.patch.object( + DeepseekV2WeightLoaderMixin, "post_load_weights" + ): + Glm5NextForConditionalGeneration.load_weights(Model(), weights) + + self.assertEqual([entry[2] for entry in loaded], ["w1", "w2", "w3"]) + self.assertEqual([entry[3] for entry in loaded], [288, 288, 288]) + self.assertEqual( + [entry[1] for entry in loaded], + [ + "model.layers.3.mlp.experts.w13_weight", + "model.layers.3.mlp.experts.w2_weight", + "model.layers.3.mlp.experts.w13_weight", + ], + ) + self.assertEqual([entry[0].item() for entry in loaded], [1, 2, 3]) + + class TestMiniMaxGates(_FusionGateCase): def test_a_config_without_shared_experts_cannot_fuse(self): from sglang.srt.models.minimax_m3 import MiniMaxM3SparseForCausalLM