[AMD] GLM-5.3-Flash: fuse shared expert and KDA projections on Quark MXFP4 (#41870)

Co-authored-by: Thomas Wang <thomawan@amd.com>
This commit is contained in:
Jacob0226
2026-10-02 20:47:47 -07:00
committed by GitHub
co-authored by Thomas Wang
parent 482b062d28
commit af1bef3eaf
5 changed files with 245 additions and 7 deletions
@@ -267,6 +267,20 @@ class QuantizationConfig(ABC):
"""
raise NotImplementedError()
def is_linear_unquantized(self, prefix: str) -> bool:
"""Whether a linear layer at ``prefix`` keeps its checkpoint precision.
A query for deciding module layouts before layers are built, so an
override must not register ``prefix`` as a quantized layer.
"""
from sglang.srt.layers.linear import LinearBase
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
return isinstance(
self.get_quant_method(LinearBase(1, 1), prefix=prefix),
UnquantizedLinearMethod,
)
@abstractmethod
def get_scaled_act_names(self) -> List[str]:
"""Returns the activation function names that should be post-scaled.
@@ -419,6 +419,15 @@ class QuarkConfig(QuantizationConfig):
packed_modules_mapping=packed_modules_mapping,
)
def is_linear_unquantized(self, prefix: str) -> bool:
# get_quant_method registers every non-excluded prefix as an
# online-quantized layer, so answer from the exclude list instead.
return self.excluded_fp8_config is None and should_ignore_layer(
prefix,
ignore=self.exclude_layers,
fused_mapping=self.packed_modules_mapping,
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> Optional["QuantizeMethodBase"]:
+51 -7
View File
@@ -41,7 +41,6 @@ from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelBatchedLinear,
ColumnParallelLinear,
LinearBase,
MergedColumnParallelLinear,
MergedColumnParallelRepeatedLinear,
QKVParallelLinear,
@@ -55,7 +54,6 @@ from sglang.srt.layers.moe.utils import (
is_shared_experts_fusion_disabled,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.layers.quantization.utils import is_layer_skipped
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -363,10 +361,10 @@ class Glm5NextLinearAttention(nn.Module):
"modelopt_fp8",
"modelopt_fp4",
"modelopt_mixed",
"quark",
}:
return False
probe = LinearBase(1, 1)
source_projs = [
proj
for fused_proj in fused_projs
@@ -375,10 +373,7 @@ class Glm5NextLinearAttention(nn.Module):
if "fused_qkvbfg_a_proj" in fused_projs:
source_projs.append("qkv_proj")
return all(
isinstance(
quant_config.get_quant_method(probe, prefix=f"{prefix}.{proj}"),
UnquantizedLinearMethod,
)
quant_config.is_linear_unquantized(f"{prefix}.{proj}")
for proj in source_projs
)
@@ -1381,6 +1376,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
"HIP shared experts fusion requires the validated "
"clamped SiLU G1U1 activation."
)
if quant_config is not None and quant_config.get_name() == "quark":
return cls._quark_mxfp4_shared_fusion_disable_reason(
text_config, quant_config
)
if (
quant_config is None
or quant_config.get_name() != "fp8"
@@ -1408,6 +1407,51 @@ class Glm5NextForConditionalGeneration(nn.Module):
)
return None
@staticmethod
def _quark_mxfp4_shared_fusion_disable_reason(text_config, quant_config):
from sglang.srt.layers.quantization.quark.utils import (
deep_compare,
should_ignore_layer,
)
reason = (
"HIP shared experts fusion requires routed and shared experts to "
"use the same Quark MXFP4 layout."
)
if not quant_config.can_fuse_shared_expert():
return reason
lookup_stub = torch.nn.Module()
first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0)
for layer_id in range(first_sparse_layer, text_config.num_hidden_layers):
moe_prefix = f"model.layers.{layer_id}.mlp"
routed_name = f"{moe_prefix}.experts"
shared_names = [
f"{moe_prefix}.shared_experts.{proj}"
for proj in ("gate_up_proj", "down_proj")
]
if any(
should_ignore_layer(
name,
ignore=quant_config.exclude_layers,
fused_mapping=quant_config.packed_modules_mapping,
)
for name in (routed_name, *shared_names)
):
return reason
try:
routed = quant_config._find_matched_config(routed_name, lookup_stub)
shared = [
quant_config._find_matched_config(name, lookup_stub)
for name in shared_names
]
except ValueError:
return reason
if not quant_config._is_mx_fp4(
routed.get("weight"), routed.get("input_tensors")
) or not all(deep_compare(routed, cfg) for cfg in shared):
return reason
return None
def determine_num_fused_shared_experts(self):
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts