[AMD] [GLM-5.3-Flash Day 0] Load the MXFP4 MTP draft layer (#39779)
Co-authored-by: Thomas Wang <thomawan@amd.com> Co-authored-by: CHANG <jacchang@crsuse2-slog-003.us-east2-a.compute.internal> Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
This commit is contained in:
co-authored by
Thomas Wang
CHANG
Kevin Mi
Claude Opus 5.5
Kevin Mi
parent
01275aab6b
commit
c19dc43cc7
@@ -869,6 +869,17 @@ class QuarkConfig(QuantizationConfig):
|
||||
if fnmatch.fnmatch(layer_name, name_pattern):
|
||||
return layer_quant_config[name_pattern]
|
||||
|
||||
# entries may name experts individually, so they resolve the fused module
|
||||
if layer_name.endswith(".experts"):
|
||||
expert_prefix = layer_name + "."
|
||||
expert_entries = {
|
||||
name[len(expert_prefix) :]: cfg
|
||||
for name, cfg in layer_quant_config.items()
|
||||
if name.startswith(expert_prefix)
|
||||
}
|
||||
if expert_entries:
|
||||
return self._fused_expert_config(layer_name, expert_entries)
|
||||
|
||||
layer_type = type(module).__name__
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
@@ -881,6 +892,44 @@ class QuarkConfig(QuantizationConfig):
|
||||
)
|
||||
return global_quant_config
|
||||
|
||||
@staticmethod
|
||||
def _fused_expert_config(
|
||||
layer_name: str, entries: dict[str, dict[str, Any]]
|
||||
) -> dict[str, Any]:
|
||||
projections_by_expert: dict[int, set[str]] = {}
|
||||
for suffix in entries:
|
||||
index, _, projection = suffix.partition(".")
|
||||
if not index.isdigit() or not projection:
|
||||
raise ValueError(
|
||||
f"Found a per-expert entry {suffix!r} in {layer_name} that is "
|
||||
"not <expert index>.<projection>."
|
||||
)
|
||||
projections_by_expert.setdefault(int(index), set()).add(projection)
|
||||
|
||||
# one fused module spans the bank, so a gap below the highest pinned index raises
|
||||
pinned = projections_by_expert.keys()
|
||||
missing = sorted(set(range(max(pinned) + 1)) - pinned)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Found per-expert entries in {layer_name} that skip experts "
|
||||
f"{missing[:4]}. SGLang requires all to use the same scheme."
|
||||
)
|
||||
|
||||
projections = next(iter(projections_by_expert.values()))
|
||||
if any(p != projections for p in projections_by_expert.values()):
|
||||
raise ValueError(
|
||||
f"Found different projections pinned per expert in {layer_name}. "
|
||||
"SGLang requires all to use the same scheme."
|
||||
)
|
||||
|
||||
configs = list(entries.values())
|
||||
if not all(deep_compare(cfg, configs[0]) for cfg in configs):
|
||||
raise ValueError(
|
||||
f"Found different quantization configurations among the experts "
|
||||
f"of {layer_name}. SGLang requires all to use the same scheme."
|
||||
)
|
||||
return configs[0]
|
||||
|
||||
def _get_scheme_from_config(self, config: dict[str, Any]) -> "QuarkLinearScheme":
|
||||
if config.get("output_tensors") or config.get("bias"):
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -25,10 +25,18 @@ class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN):
|
||||
@classmethod
|
||||
def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper:
|
||||
text_config = getattr(config, "text_config", config)
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
f"model.layers.{text_config.num_hidden_layers}": "model.decoder",
|
||||
},
|
||||
n = text_config.num_hidden_layers
|
||||
# lookups arrive as checkpoint and normalized names, so every rule has both forms
|
||||
draft_rules: dict[str, str] = {}
|
||||
for ckpt_prefix in (f"model.language_model.layers.{n}", f"model.layers.{n}"):
|
||||
# eh_proj/enorm/hnorm sit beside the block under `model`, not in `decoder`
|
||||
draft_rules[f"{ckpt_prefix}.eh_proj"] = "model.eh_proj"
|
||||
draft_rules[f"{ckpt_prefix}.enorm"] = "model.enorm"
|
||||
draft_rules[f"{ckpt_prefix}.hnorm"] = "model.hnorm"
|
||||
draft_rules[ckpt_prefix] = "model.decoder"
|
||||
# target rules still normalize the non-draft layers and vision tower in `exclude`
|
||||
return Glm5NextForConditionalGeneration.hf_to_sglang_mapper | WeightsMapper(
|
||||
orig_to_new_substr=draft_rules,
|
||||
)
|
||||
|
||||
def _resolve_nextn_quant_config(self, config, quant_config):
|
||||
|
||||
@@ -201,6 +201,74 @@ class TestMixedPrecisionLayerConfig(CustomTestCase):
|
||||
self.assertIsNone(_mixed_precision_layer_map({"quant_algo": "NVFP4"}))
|
||||
|
||||
|
||||
class TestFusedExpertConfig(CustomTestCase):
|
||||
"""Per-expert-only entries must not leave a FusedMoE on the global scheme."""
|
||||
|
||||
_FP8 = {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"block_size": [128, 128],
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 128,
|
||||
},
|
||||
}
|
||||
_MXFP4 = {
|
||||
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32},
|
||||
"input_tensors": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32},
|
||||
}
|
||||
_PREFIX = "model.decoder.mlp.experts"
|
||||
|
||||
def _entries(self, num_experts):
|
||||
return {
|
||||
f"{index}.{projection}": deepcopy(self._FP8)
|
||||
for index in range(num_experts)
|
||||
for projection in ("gate_proj", "up_proj", "down_proj")
|
||||
}
|
||||
|
||||
def _config_with(self, layer_quant_config):
|
||||
quark_config = _bare_config()
|
||||
quark_config.quant_config = {
|
||||
"layer_quant_config": layer_quant_config,
|
||||
"layer_type_quant_config": {},
|
||||
"global_quant_config": self._MXFP4,
|
||||
}
|
||||
quark_config.packed_modules_mapping = {}
|
||||
return quark_config
|
||||
|
||||
def _lookup(self, entries):
|
||||
quark_config = self._config_with(
|
||||
{f"{self._PREFIX}.{s}": cfg for s, cfg in entries.items()}
|
||||
)
|
||||
return quark_config._find_matched_config(self._PREFIX, torch.nn.Module())
|
||||
|
||||
def test_fused_moe_resolves_from_per_expert_entries(self):
|
||||
matched = self._lookup(self._entries(288))
|
||||
self.assertEqual(matched["weight"]["dtype"], "fp8_e4m3")
|
||||
|
||||
def test_incomplete_or_mixed_expert_coverage_raises(self):
|
||||
missing = {s: c for s, c in self._entries(4).items() if not s.startswith("2.")}
|
||||
conflicting = self._entries(4)
|
||||
conflicting["2.up_proj"] = deepcopy(self._MXFP4)
|
||||
partial_projections = self._entries(4)
|
||||
del partial_projections["3.down_proj"]
|
||||
cases = {
|
||||
"missing_expert": (missing, "skip experts"),
|
||||
"conflicting_scheme": (conflicting, "different quantization"),
|
||||
"partial_projections": (partial_projections, "different projections"),
|
||||
"fused_param_name": ({"w13_weight_scale": self._FP8}, "expert index"),
|
||||
}
|
||||
for name, (entries, message) in cases.items():
|
||||
with self.subTest(name), self.assertRaisesRegex(ValueError, message):
|
||||
self._lookup(entries)
|
||||
|
||||
def test_experts_without_per_expert_entries_fall_through_to_global(self):
|
||||
self.assertEqual(self._lookup({})["weight"]["dtype"], "fp4")
|
||||
|
||||
|
||||
class TestParseNvfp4Excludes(CustomTestCase):
|
||||
"""ModelOpt `ignore` lists mix `re:`-prefixed regexes with fnmatch globs."""
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from unittest.mock import patch
|
||||
from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config
|
||||
from sglang.srt.models import glm5_next
|
||||
from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration
|
||||
from sglang.srt.models.glm5_next_nextn import Glm5NextForConditionalGenerationNextN
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -99,6 +100,23 @@ class TestGlm5NextModelOpt(CustomTestCase):
|
||||
|
||||
self.assertIsNone(reason)
|
||||
|
||||
def test_nextn_mapper_routes_checkpoint_names_to_runtime_modules(self):
|
||||
"""Draft quant lookups miss unless checkpoint names reach NextN module paths."""
|
||||
mapper = Glm5NextForConditionalGenerationNextN.get_hf_to_sglang_mapper(
|
||||
self._hf_config()
|
||||
)
|
||||
cases = {
|
||||
"model.language_model.layers.45.mlp.experts.7.up_proj": "model.decoder.mlp.experts.7.up_proj",
|
||||
"model.language_model.layers.45.eh_proj.weight": "model.eh_proj.weight",
|
||||
"model.layers.45.eh_proj.weight": "model.eh_proj.weight",
|
||||
"model.language_model.layers.45.enorm.weight": "model.enorm.weight",
|
||||
"model.language_model.layers.44.mlp.experts.3.up_proj": "model.layers.44.mlp.experts.3.up_proj",
|
||||
"model.visual.blocks.0.attn.proj": "visual.blocks.0.attn.proj",
|
||||
}
|
||||
for source, expected in cases.items():
|
||||
with self.subTest(source=source):
|
||||
self.assertEqual(mapper._map_name(source), expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user