From af1bef3eaf723babb83df1f0e25f8cca2c5b9e58 Mon Sep 17 00:00:00 2001 From: Jacob0226 Date: Sat, 3 Oct 2026 11:47:47 +0800 Subject: [PATCH] [AMD] GLM-5.3-Flash: fuse shared expert and KDA projections on Quark MXFP4 (#41870) Co-authored-by: Thomas Wang --- .../srt/layers/quantization/base_config.py | 14 +++ .../srt/layers/quantization/quark/quark.py | 9 ++ python/sglang/srt/models/glm5_next.py | 58 ++++++++-- .../unit/models/test_glm5_next_bfg_fusion.py | 65 +++++++++++ .../test_shared_experts_fusion_gates.py | 106 ++++++++++++++++++ 5 files changed, 245 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/layers/quantization/base_config.py b/python/sglang/srt/layers/quantization/base_config.py index 6ab29d05d6..a080d0c19d 100644 --- a/python/sglang/srt/layers/quantization/base_config.py +++ b/python/sglang/srt/layers/quantization/base_config.py @@ -267,6 +267,20 @@ class QuantizationConfig(ABC): """ raise NotImplementedError() + def is_linear_unquantized(self, prefix: str) -> bool: + """Whether a linear layer at ``prefix`` keeps its checkpoint precision. + + A query for deciding module layouts before layers are built, so an + override must not register ``prefix`` as a quantized layer. + """ + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod + + return isinstance( + self.get_quant_method(LinearBase(1, 1), prefix=prefix), + UnquantizedLinearMethod, + ) + @abstractmethod def get_scaled_act_names(self) -> List[str]: """Returns the activation function names that should be post-scaled. diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 9b848e9919..943005c79a 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -419,6 +419,15 @@ class QuarkConfig(QuantizationConfig): packed_modules_mapping=packed_modules_mapping, ) + def is_linear_unquantized(self, prefix: str) -> bool: + # get_quant_method registers every non-excluded prefix as an + # online-quantized layer, so answer from the exclude list instead. + return self.excluded_fp8_config is None and should_ignore_layer( + prefix, + ignore=self.exclude_layers, + fused_mapping=self.packed_modules_mapping, + ) + def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> Optional["QuantizeMethodBase"]: diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index d8f3f76286..43cc6dcf27 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -41,7 +41,6 @@ from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, ColumnParallelLinear, - LinearBase, MergedColumnParallelLinear, MergedColumnParallelRepeatedLinear, QKVParallelLinear, @@ -55,7 +54,6 @@ from sglang.srt.layers.moe.utils import ( is_shared_experts_fusion_disabled, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod from sglang.srt.layers.quantization.utils import is_layer_skipped from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -363,10 +361,10 @@ class Glm5NextLinearAttention(nn.Module): "modelopt_fp8", "modelopt_fp4", "modelopt_mixed", + "quark", }: return False - probe = LinearBase(1, 1) source_projs = [ proj for fused_proj in fused_projs @@ -375,10 +373,7 @@ class Glm5NextLinearAttention(nn.Module): if "fused_qkvbfg_a_proj" in fused_projs: source_projs.append("qkv_proj") return all( - isinstance( - quant_config.get_quant_method(probe, prefix=f"{prefix}.{proj}"), - UnquantizedLinearMethod, - ) + quant_config.is_linear_unquantized(f"{prefix}.{proj}") for proj in source_projs ) @@ -1381,6 +1376,10 @@ class Glm5NextForConditionalGeneration(nn.Module): "HIP shared experts fusion requires the validated " "clamped SiLU G1U1 activation." ) + if quant_config is not None and quant_config.get_name() == "quark": + return cls._quark_mxfp4_shared_fusion_disable_reason( + text_config, quant_config + ) if ( quant_config is None or quant_config.get_name() != "fp8" @@ -1408,6 +1407,51 @@ class Glm5NextForConditionalGeneration(nn.Module): ) return None + @staticmethod + def _quark_mxfp4_shared_fusion_disable_reason(text_config, quant_config): + from sglang.srt.layers.quantization.quark.utils import ( + deep_compare, + should_ignore_layer, + ) + + reason = ( + "HIP shared experts fusion requires routed and shared experts to " + "use the same Quark MXFP4 layout." + ) + if not quant_config.can_fuse_shared_expert(): + return reason + lookup_stub = torch.nn.Module() + first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0) + for layer_id in range(first_sparse_layer, text_config.num_hidden_layers): + moe_prefix = f"model.layers.{layer_id}.mlp" + routed_name = f"{moe_prefix}.experts" + shared_names = [ + f"{moe_prefix}.shared_experts.{proj}" + for proj in ("gate_up_proj", "down_proj") + ] + if any( + should_ignore_layer( + name, + ignore=quant_config.exclude_layers, + fused_mapping=quant_config.packed_modules_mapping, + ) + for name in (routed_name, *shared_names) + ): + return reason + try: + routed = quant_config._find_matched_config(routed_name, lookup_stub) + shared = [ + quant_config._find_matched_config(name, lookup_stub) + for name in shared_names + ] + except ValueError: + return reason + if not quant_config._is_mx_fp4( + routed.get("weight"), routed.get("input_tensors") + ) or not all(deep_compare(routed, cfg) for cfg in shared): + return reason + return None + def determine_num_fused_shared_experts(self): self.num_fused_shared_experts = ( 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts diff --git a/test/registered/unit/models/test_glm5_next_bfg_fusion.py b/test/registered/unit/models/test_glm5_next_bfg_fusion.py index d3b3411a55..db34e2c1ab 100644 --- a/test/registered/unit/models/test_glm5_next_bfg_fusion.py +++ b/test/registered/unit/models/test_glm5_next_bfg_fusion.py @@ -5,6 +5,7 @@ from unittest.mock import patch import torch import torch.nn.functional as F +from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod from sglang.srt.models import glm5_next from sglang.srt.runtime_context import get_context, get_parallel @@ -30,6 +31,8 @@ class MockFp8Config: def __init__(self, ignored): self.ignored_layers = {f"{PREFIX}.{name}" for name in ignored} + is_linear_unquantized = QuantizationConfig.is_linear_unquantized + def get_name(self): return "fp8" @@ -44,6 +47,35 @@ class MockFp8Config: return MockQuantizedLinearMethod() +def quark_mxfp4_config(excluded): + """A Quark MXFP4 export that keeps `excluded` in BF16, named as the HF + checkpoint names them and mapped the way the loader maps them.""" + from sglang.srt.layers.quantization.quark.quark import QuarkConfig + + mxfp4 = {"dtype": "fp4", "qscheme": "per_group", "group_size": 32} + model = glm5_next.Glm5NextForConditionalGeneration + quant = QuarkConfig.from_config( + { + "quant_method": "quark", + "export": {"kv_cache_group": [], "pack_method": "reorder"}, + "global_quant_config": { + "weight": {**mxfp4, "is_dynamic": False, "scale_format": "e8m0"}, + "input_tensors": {**mxfp4, "is_dynamic": True, "scale_format": "e8m0"}, + "output_tensors": None, + "bias": None, + }, + "layer_quant_config": {}, + "layer_type_quant_config": {}, + "exclude": [ + f"model.language_model.layers.0.self_attn.{name}" for name in excluded + ], + "packed_modules_mapping": dict(model.packed_modules_mapping), + } + ) + quant.apply_weight_name_mapper(model.hf_to_sglang_mapper) + return quant + + class TestGlm5NextBfgFusion(unittest.TestCase): def setUp(self): self.addCleanup(torch.set_default_dtype, torch.get_default_dtype()) @@ -167,6 +199,39 @@ class TestGlm5NextBfgFusion(unittest.TestCase): ) ) + def test_quark_fuses_only_projections_the_checkpoint_keeps_in_bf16(self): + for packed in ("fused_qkvbfg_a_proj", "fused_bfg_a_proj"): + with self.subTest(packed=packed): + self.assertTrue( + glm5_next.Glm5NextLinearAttention._can_fuse_proj( + quark_mxfp4_config(QKV + BFG), + PREFIX, + packed, + "fused_fg_b_proj", + ) + ) + for quantized in QKV + BFG: + quant = quark_mxfp4_config( + [name for name in QKV + BFG if name != quantized] + ) + with self.subTest(quantized=quantized): + self.assertFalse( + glm5_next.Glm5NextLinearAttention._can_fuse_proj( + quant, PREFIX, "fused_qkvbfg_a_proj", "fused_fg_b_proj" + ) + ) + # The probe must not register a quantized layer the model never builds. + self.assertEqual(quant.quantized_layers, ([], 0)) + + def test_quark_answers_from_its_exclude_list(self): + quant = quark_mxfp4_config(["b_proj"]) + self.assertTrue(quant.is_linear_unquantized(f"{PREFIX}.b_proj")) + self.assertFalse(quant.is_linear_unquantized(f"{PREFIX}.q_proj")) + self.assertEqual(quant.quantized_layers, ([], 0)) + # Excluded layers that a mixed-precision export keeps in FP8 are quantized. + with patch.object(quant, "excluded_fp8_config", SimpleNamespace()): + self.assertFalse(quant.is_linear_unquantized(f"{PREFIX}.b_proj")) + def test_lora_disables_full_and_bfg_fusion(self): for enable_lora, paths in ((True, None), (False, ["adapter"])): with patch.object( diff --git a/test/registered/unit/models/test_shared_experts_fusion_gates.py b/test/registered/unit/models/test_shared_experts_fusion_gates.py index 273136c0fc..bbd1ce6201 100644 --- a/test/registered/unit/models/test_shared_experts_fusion_gates.py +++ b/test/registered/unit/models/test_shared_experts_fusion_gates.py @@ -34,6 +34,42 @@ def _quant(name: str): return SimpleNamespace(get_name=lambda: name) +_QUARK_MXFP4 = { + "weight": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "is_dynamic": False, + "scale_format": "e8m0", + }, + "input_tensors": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "is_dynamic": True, + "scale_format": "e8m0", + }, + "output_tensors": None, + "bias": None, +} +_QUARK_BLOCK_FP8 = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [128, 128], + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 128, + "is_dynamic": True, + }, + "output_tensors": None, + "bias": None, +} + + def _import_bailing_modules(): if importlib.util.find_spec("vllm") is not None: from sglang.srt.models import bailing_moe_nextn, bailing_moe_v3 @@ -366,6 +402,76 @@ class TestGlm5NextGate(_FusionGateCase): self._reason_on_gfx950(quant=quant), ) + def _quark( + self, *, exclude=(), fp8_experts=(45,), fp8_shared=(45,), fp8_shared_down=() + ): + """A Quark export shaped like amd/GLM-5.3-Flash-Quark-MXFP4: MXFP4 by + default, layer 45 (the MTP draft) pinned to block-FP8 expert by expert, + and its names mapped the way the loader maps them.""" + from sglang.srt.layers.quantization.quark.quark import QuarkConfig + from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration + + layer_quant_config = {} + for layer in fp8_experts: + for expert in range(288): + for proj in ("gate_proj", "up_proj", "down_proj"): + name = f"model.language_model.layers.{layer}.mlp.experts.{expert}.{proj}" + layer_quant_config[name] = _QUARK_BLOCK_FP8 + for layer in fp8_shared: + for proj in ("gate_proj", "up_proj", "down_proj"): + name = f"model.language_model.layers.{layer}.mlp.shared_experts.{proj}" + layer_quant_config[name] = _QUARK_BLOCK_FP8 + for layer in fp8_shared_down: + name = f"model.language_model.layers.{layer}.mlp.shared_experts.down_proj" + layer_quant_config[name] = _QUARK_BLOCK_FP8 + quant = QuarkConfig.from_config( + { + "quant_method": "quark", + "export": {"kv_cache_group": [], "pack_method": "reorder"}, + "global_quant_config": _QUARK_MXFP4, + "layer_quant_config": layer_quant_config, + "layer_type_quant_config": {}, + "exclude": [f"model.language_model.layers.{name}" for name in exclude], + "packed_modules_mapping": { + **Glm5NextForConditionalGeneration.packed_modules_mapping, + "gate_up_proj": ["gate_proj", "up_proj"], + }, + } + ) + quant.apply_weight_name_mapper( + Glm5NextForConditionalGeneration.hf_to_sglang_mapper + ) + return quant + + def test_quark_mxfp4_is_admitted(self): + config = self._config(num_hidden_layers=45) + self.assertIsNone(self._reason_on_gfx950(config=config, quant=self._quark())) + + def test_quark_target_layer_experts_must_share_one_mxfp4_spec(self): + config = self._config(num_hidden_layers=45) + cases = { + "excluded routed expert": self._quark( + exclude=["10.mlp.experts.5.gate_proj"] + ), + "excluded shared expert": self._quark( + exclude=[ + "10.mlp.shared_experts.gate_proj", + "10.mlp.shared_experts.up_proj", + ] + ), + "block-FP8 routed and shared experts": self._quark( + fp8_experts=(10, 45), fp8_shared=(10, 45) + ), + "block-FP8 shared expert only": self._quark(fp8_shared=(10, 45)), + "block-FP8 shared down_proj only": self._quark(fp8_shared_down=(10,)), + } + for name, quant in cases.items(): + with self.subTest(name): + self.assertIn( + "same Quark MXFP4 layout", + self._reason_on_gfx950(config=config, quant=quant), + ) + def test_ep_and_a2a_topologies_are_rejected(self): import sglang.srt.models.glm5_next as glm5_next from sglang.srt.layers.moe.utils import MoeA2ABackend