[AMD] [GLM-5.3-Flash Day 0] Load the MXFP4 MTP draft layer (#39779)
Co-authored-by: Thomas Wang <thomawan@amd.com> Co-authored-by: CHANG <jacchang@crsuse2-slog-003.us-east2-a.compute.internal> Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
This commit is contained in:
co-authored by
Thomas Wang
CHANG
Kevin Mi
Claude Opus 5.5
Kevin Mi
parent
01275aab6b
commit
c19dc43cc7
@@ -869,6 +869,17 @@ class QuarkConfig(QuantizationConfig):
|
||||
if fnmatch.fnmatch(layer_name, name_pattern):
|
||||
return layer_quant_config[name_pattern]
|
||||
|
||||
# entries may name experts individually, so they resolve the fused module
|
||||
if layer_name.endswith(".experts"):
|
||||
expert_prefix = layer_name + "."
|
||||
expert_entries = {
|
||||
name[len(expert_prefix) :]: cfg
|
||||
for name, cfg in layer_quant_config.items()
|
||||
if name.startswith(expert_prefix)
|
||||
}
|
||||
if expert_entries:
|
||||
return self._fused_expert_config(layer_name, expert_entries)
|
||||
|
||||
layer_type = type(module).__name__
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
@@ -881,6 +892,44 @@ class QuarkConfig(QuantizationConfig):
|
||||
)
|
||||
return global_quant_config
|
||||
|
||||
@staticmethod
|
||||
def _fused_expert_config(
|
||||
layer_name: str, entries: dict[str, dict[str, Any]]
|
||||
) -> dict[str, Any]:
|
||||
projections_by_expert: dict[int, set[str]] = {}
|
||||
for suffix in entries:
|
||||
index, _, projection = suffix.partition(".")
|
||||
if not index.isdigit() or not projection:
|
||||
raise ValueError(
|
||||
f"Found a per-expert entry {suffix!r} in {layer_name} that is "
|
||||
"not <expert index>.<projection>."
|
||||
)
|
||||
projections_by_expert.setdefault(int(index), set()).add(projection)
|
||||
|
||||
# one fused module spans the bank, so a gap below the highest pinned index raises
|
||||
pinned = projections_by_expert.keys()
|
||||
missing = sorted(set(range(max(pinned) + 1)) - pinned)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Found per-expert entries in {layer_name} that skip experts "
|
||||
f"{missing[:4]}. SGLang requires all to use the same scheme."
|
||||
)
|
||||
|
||||
projections = next(iter(projections_by_expert.values()))
|
||||
if any(p != projections for p in projections_by_expert.values()):
|
||||
raise ValueError(
|
||||
f"Found different projections pinned per expert in {layer_name}. "
|
||||
"SGLang requires all to use the same scheme."
|
||||
)
|
||||
|
||||
configs = list(entries.values())
|
||||
if not all(deep_compare(cfg, configs[0]) for cfg in configs):
|
||||
raise ValueError(
|
||||
f"Found different quantization configurations among the experts "
|
||||
f"of {layer_name}. SGLang requires all to use the same scheme."
|
||||
)
|
||||
return configs[0]
|
||||
|
||||
def _get_scheme_from_config(self, config: dict[str, Any]) -> "QuarkLinearScheme":
|
||||
if config.get("output_tensors") or config.get("bias"):
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -25,10 +25,18 @@ class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN):
|
||||
@classmethod
|
||||
def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper:
|
||||
text_config = getattr(config, "text_config", config)
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
f"model.layers.{text_config.num_hidden_layers}": "model.decoder",
|
||||
},
|
||||
n = text_config.num_hidden_layers
|
||||
# lookups arrive as checkpoint and normalized names, so every rule has both forms
|
||||
draft_rules: dict[str, str] = {}
|
||||
for ckpt_prefix in (f"model.language_model.layers.{n}", f"model.layers.{n}"):
|
||||
# eh_proj/enorm/hnorm sit beside the block under `model`, not in `decoder`
|
||||
draft_rules[f"{ckpt_prefix}.eh_proj"] = "model.eh_proj"
|
||||
draft_rules[f"{ckpt_prefix}.enorm"] = "model.enorm"
|
||||
draft_rules[f"{ckpt_prefix}.hnorm"] = "model.hnorm"
|
||||
draft_rules[ckpt_prefix] = "model.decoder"
|
||||
# target rules still normalize the non-draft layers and vision tower in `exclude`
|
||||
return Glm5NextForConditionalGeneration.hf_to_sglang_mapper | WeightsMapper(
|
||||
orig_to_new_substr=draft_rules,
|
||||
)
|
||||
|
||||
def _resolve_nextn_quant_config(self, config, quant_config):
|
||||
|
||||
Reference in New Issue
Block a user