[AMD] [GLM-5.3-Flash Day 0] Load the MXFP4 MTP draft layer (#39779)

Co-authored-by: Thomas Wang <thomawan@amd.com>
Co-authored-by: CHANG <jacchang@crsuse2-slog-003.us-east2-a.compute.internal>
Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
This commit is contained in:
Jacob0226
2026-09-22 15:07:51 -07:00
committed by GitHub
co-authored by Thomas Wang CHANG Kevin Mi Claude Opus 5.5 Kevin Mi
parent 01275aab6b
commit c19dc43cc7
4 changed files with 147 additions and 4 deletions
@@ -869,6 +869,17 @@ class QuarkConfig(QuantizationConfig):
if fnmatch.fnmatch(layer_name, name_pattern):
return layer_quant_config[name_pattern]
# entries may name experts individually, so they resolve the fused module
if layer_name.endswith(".experts"):
expert_prefix = layer_name + "."
expert_entries = {
name[len(expert_prefix) :]: cfg
for name, cfg in layer_quant_config.items()
if name.startswith(expert_prefix)
}
if expert_entries:
return self._fused_expert_config(layer_name, expert_entries)
layer_type = type(module).__name__
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
@@ -881,6 +892,44 @@ class QuarkConfig(QuantizationConfig):
)
return global_quant_config
@staticmethod
def _fused_expert_config(
layer_name: str, entries: dict[str, dict[str, Any]]
) -> dict[str, Any]:
projections_by_expert: dict[int, set[str]] = {}
for suffix in entries:
index, _, projection = suffix.partition(".")
if not index.isdigit() or not projection:
raise ValueError(
f"Found a per-expert entry {suffix!r} in {layer_name} that is "
"not <expert index>.<projection>."
)
projections_by_expert.setdefault(int(index), set()).add(projection)
# one fused module spans the bank, so a gap below the highest pinned index raises
pinned = projections_by_expert.keys()
missing = sorted(set(range(max(pinned) + 1)) - pinned)
if missing:
raise ValueError(
f"Found per-expert entries in {layer_name} that skip experts "
f"{missing[:4]}. SGLang requires all to use the same scheme."
)
projections = next(iter(projections_by_expert.values()))
if any(p != projections for p in projections_by_expert.values()):
raise ValueError(
f"Found different projections pinned per expert in {layer_name}. "
"SGLang requires all to use the same scheme."
)
configs = list(entries.values())
if not all(deep_compare(cfg, configs[0]) for cfg in configs):
raise ValueError(
f"Found different quantization configurations among the experts "
f"of {layer_name}. SGLang requires all to use the same scheme."
)
return configs[0]
def _get_scheme_from_config(self, config: dict[str, Any]) -> "QuarkLinearScheme":
if config.get("output_tensors") or config.get("bias"):
raise NotImplementedError(
+12 -4
View File
@@ -25,10 +25,18 @@ class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN):
@classmethod
def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper:
text_config = getattr(config, "text_config", config)
return WeightsMapper(
orig_to_new_substr={
f"model.layers.{text_config.num_hidden_layers}": "model.decoder",
},
n = text_config.num_hidden_layers
# lookups arrive as checkpoint and normalized names, so every rule has both forms
draft_rules: dict[str, str] = {}
for ckpt_prefix in (f"model.language_model.layers.{n}", f"model.layers.{n}"):
# eh_proj/enorm/hnorm sit beside the block under `model`, not in `decoder`
draft_rules[f"{ckpt_prefix}.eh_proj"] = "model.eh_proj"
draft_rules[f"{ckpt_prefix}.enorm"] = "model.enorm"
draft_rules[f"{ckpt_prefix}.hnorm"] = "model.hnorm"
draft_rules[ckpt_prefix] = "model.decoder"
# target rules still normalize the non-draft layers and vision tower in `exclude`
return Glm5NextForConditionalGeneration.hf_to_sglang_mapper | WeightsMapper(
orig_to_new_substr=draft_rules,
)
def _resolve_nextn_quant_config(self, config, quant_config):