diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 5acbbebb6..b96eee133 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -110,6 +110,7 @@ def is_deepseek_dsa(config) -> bool: "MistralLarge3ForCausalLM", "PixtralForConditionalGeneration", "GlmMoeDsaForCausalLM", + "GlmMoeDsaForCausalLMNextN", "LongcatFlashForCausalLM", "LongcatFlashForCausalLMNextN", ) @@ -536,10 +537,12 @@ class ModelConfig: if is_draft_model and self.hf_config.architectures[0] in [ "DeepseekV3ForCausalLM", "DeepseekV32ForCausalLM", - "GlmMoeDsaForCausalLM", ]: self.hf_config.architectures[0] = "DeepseekV3ForCausalLMNextN" + if is_draft_model and self.hf_config.architectures[0] == "GlmMoeDsaForCausalLM": + self.hf_config.architectures[0] = "GlmMoeDsaForCausalLMNextN" + if ( is_draft_model and self.hf_config.architectures[0] == "DeepseekV4ForCausalLM" @@ -741,6 +744,7 @@ class ModelConfig: or "Glm4MoeLiteForCausalLM" in self.hf_config.architectures or "Glm4MoeLiteForCausalLMNextN" in self.hf_config.architectures or "GlmMoeDsaForCausalLM" in self.hf_config.architectures + or "GlmMoeDsaForCausalLMNextN" in self.hf_config.architectures or "LongcatFlashForCausalLM" in self.hf_config.architectures or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures or "DotsVLMForCausalLM" in self.hf_config.architectures diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index c5b4939c8..fb98511f1 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -287,6 +287,18 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): }, ) + def _resolve_nextn_quant_config(self, config, quant_config): + if quant_config is None or quant_config.get_name() != "quark": + return quant_config + + from sglang.srt.layers.quantization.quark.utils import should_ignore_layer + + ckpt_prefix = f"model.layers.{config.num_hidden_layers}" + mapped_prefix = self.hf_to_sglang_mapper._map_name(ckpt_prefix) + if should_ignore_layer(mapped_prefix, quant_config.exclude_layers): + return None + return quant_config + def __init__( self, config: PretrainedConfig, @@ -310,17 +322,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): self.cp_rank = None self.cp_size = None - nextn_quant_config = quant_config - # For quark, if the MTP layer is listed in exclude_layers, set quant_config to None. - if nextn_quant_config is not None and nextn_quant_config.get_name() == "quark": - from sglang.srt.layers.quantization.quark.utils import ( - should_ignore_layer, - ) - - ckpt_prefix = f"model.layers.{config.num_hidden_layers}" - mapped_prefix = self.hf_to_sglang_mapper._map_name(ckpt_prefix) - if should_ignore_layer(mapped_prefix, nextn_quant_config.exclude_layers): - nextn_quant_config = None + nextn_quant_config = self._resolve_nextn_quant_config(config, quant_config) self.model = DeepseekModelNextN( config, nextn_quant_config, prefix=add_prefix("model", prefix) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 1994e1249..2cfceb612 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -80,9 +80,14 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.deepseek_nextn import DeepseekV3ForCausalLMNextN from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM -from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_parallel, get_server_args, get_stream +from sglang.srt.models.utils import WeightsMapper, apply_qk_norm +from sglang.srt.runtime_context import ( + get_parallel, + get_server_args, + get_stream, +) from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -1484,4 +1489,72 @@ class GlmMoeDsaForCausalLM(DeepseekV2ForCausalLM): super().determine_num_fused_shared_experts("GlmMoeDsaForCausalLM") -EntryClass = [Glm4MoeForCausalLM, GlmMoeDsaForCausalLM] +class GlmMoeDsaForCausalLMNextN(DeepseekV3ForCausalLMNextN): + # GLM-5.2's MTP layer index differs from DeepSeek's (61), so the inherited + # substr mapping would wrongly rewrite GLM's real layer-61 weights. + # exclude_layers remapping for the MTP layer is handled explicitly in + # _resolve_nextn_quant_config below instead. + hf_to_sglang_mapper = WeightsMapper() + + _NEXTN_SPEC_WEIGHT_NAMES = ("shared_head.norm", "eh_proj", "enorm", "hnorm") + + @classmethod + def _map_mtp_ckpt_name(cls, name: str, layer_prefix: str) -> str: + # Keep this mapping in sync with DeepseekV2WeightLoaderMixin's + # NextN rule: MTP-specific weights live under model.*, while the + # decoder block weights live under model.decoder.*. + if any(part in name for part in cls._NEXTN_SPEC_WEIGHT_NAMES): + return name.replace(layer_prefix, "model", 1) + return name.replace(layer_prefix, "model.decoder", 1) + + def _resolve_nextn_quant_config(self, config, quant_config): + if quant_config is None or quant_config.get_name() != "quark": + return quant_config + + layer_prefix = f"model.layers.{config.num_hidden_layers}" + + # Quark's per-module scheme selection (e.g. MTP self_attn in PTPC-FP8 + # while MTP MoE is MXFP4) is keyed by "layer_quant_config" patterns + # using the checkpoint's "model.layers..*" naming. SGLang queries + # schemes by the runtime "model.*"/"model.decoder.*" prefix, so those + # keys need the same remap as exclude_layers below, or they silently + # fall back to the wrong (layer-type/global) scheme. + layer_quant_config = quant_config.quant_config.get("layer_quant_config") + if layer_quant_config: + quant_config.quant_config["layer_quant_config"] = { + ( + self._map_mtp_ckpt_name(pattern, layer_prefix) + if pattern.startswith(layer_prefix + ".") + else pattern + ): pattern_config + for pattern, pattern_config in layer_quant_config.items() + } + + mtp_excluded = [ + name + for name in quant_config.exclude_layers + if name.startswith(layer_prefix + ".") + ] + if not mtp_excluded: + return quant_config + + names = set(quant_config.exclude_layers) + for name in mtp_excluded: + names.add(self._map_mtp_ckpt_name(name, layer_prefix)) + + # Fused routed experts are queried by the coarse module prefix + # "model.decoder.mlp.experts". Expanded per-expert leaf excludes do not + # match that prefix, so add the coarse prefix when any routed expert in + # the MTP layer is excluded. This keeps only that fused MoE module bf16 + # while allowing the remaining draft modules to use their quant config. + if any(".mlp.experts." in name for name in mtp_excluded): + names.add("model.decoder.mlp.experts") + + import copy + + quant_config = copy.copy(quant_config) + quant_config.exclude_layers = list(names) + return quant_config + + +EntryClass = [Glm4MoeForCausalLM, GlmMoeDsaForCausalLM, GlmMoeDsaForCausalLMNextN]