[Fix] EXAONE and ERNIE 4.5 VL MoE architecture, backend and PP issues (#42303)

This commit is contained in:
Cheng Wan
2026-10-03 02:18:33 -07:00
committed by GitHub
parent be20e7453c
commit f89ead192f
8 changed files with 65 additions and 10 deletions
+5 -1
View File
@@ -636,7 +636,11 @@ def handle_model_specific_adjustments(server_args: Any):
# The prefill attention backend default + validation moved to the
# override registry (arg_groups/overrides.py: _moss_vl_overrides).
pass
elif model_arch in ["Exaone4ForCausalLM", "ExaoneMoEForCausalLM"]:
elif model_arch in [
"Exaone4ForCausalLM",
"ExaoneMoEForCausalLM",
"ExaoneMoeForCausalLM",
]:
if hf_config.sliding_window_pattern is not None:
# disable_hybrid_swa_memory moved to the override registry
# (arg_groups/overrides.py: _exaone_overrides).
@@ -12,6 +12,7 @@ from sglang.srt.arg_groups.model_overrides import bailing_moe_v3 # noqa: F401
from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401
from sglang.srt.arg_groups.model_overrides import deepseek_v2 # noqa: F401
from sglang.srt.arg_groups.model_overrides import deepseek_v4 # noqa: F401
from sglang.srt.arg_groups.model_overrides import ernie45_vl # noqa: F401
from sglang.srt.arg_groups.model_overrides import exaone # noqa: F401
from sglang.srt.arg_groups.model_overrides import falcon_h1 # noqa: F401
from sglang.srt.arg_groups.model_overrides import gemma2_gemma3 # noqa: F401
@@ -0,0 +1,19 @@
"""Config-time override declarations for ernie45_vl."""
from typing import Any
from sglang.srt.arg_groups.model_override_base import (
_register_for,
resolving_view,
)
from sglang.srt.runtime_context import attn_dp_enabled_of
@_register_for("Ernie4_5_VLMoeForConditionalGeneration")
def _ernie45_vl_overrides(server_args: Any, hf_config: Any) -> dict:
if attn_dp_enabled_of(resolving_view(server_args)):
raise ValueError(
"ERNIE 4.5 VL MoE does not support DP attention: its attention "
"heads are split over the full TP group."
)
return {}
@@ -1,20 +1,30 @@
"""Config-time override declarations for exaone."""
import logging
from typing import Any
from typing import Any, Dict
from sglang.srt.arg_groups.model_override_base import (
_register_for,
resolving_view,
)
from sglang.srt.runtime_context import get_platform
logger = logging.getLogger(__name__)
@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM")
@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
def _exaone_overrides(server_args: Any, hf_config: Any) -> dict:
overrides: Dict[str, Any] = {}
if hf_config.sliding_window_pattern is not None:
logger.warning(
f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported."
)
return {"disable_hybrid_swa_memory": True}
return {}
overrides["disable_hybrid_swa_memory"] = True
if resolving_view(server_args).attention_backend is None:
if get_platform().is_cuda and get_platform().is_sm100:
overrides["attention_backend"] = "trtllm_mha"
elif get_platform().is_cuda and get_platform().device_sm >= 80:
overrides["attention_backend"] = "fa3"
else:
overrides["attention_backend"] = "triton"
return overrides
+4 -1
View File
@@ -1030,7 +1030,10 @@ class ModelConfig:
self.hf_config.num_nextn_predict_layers = 1
self.hf_text_config.num_nextn_predict_layers = 1
if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM":
if is_draft_model and self.hf_config.architectures[0] in (
"ExaoneMoEForCausalLM",
"ExaoneMoeForCausalLM",
):
self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
self.hf_config.num_nextn_predict_layers = 1
@@ -25,7 +25,6 @@ from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -479,7 +478,6 @@ class Ernie4_5_VLMoeModel(nn.Module):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
enable_tp=not is_dp_attention_enabled(),
prefix=add_prefix("embed_tokens", prefix),
)
else:
+10 -1
View File
@@ -435,7 +435,7 @@ class Exaone4ForCausalLM(nn.Module):
self.model = self._init_model(config, quant_config, add_prefix("model", prefix))
# Exaone-4.0 32B set tie_word_embeddins to False
# Exaone-4.0 1.2B set tie_word_embeddins to True
if config.tie_word_embeddings:
if config.tie_word_embeddings and self.pp_group.world_size == 1:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
@@ -575,6 +575,15 @@ class Exaone4ForCausalLM(nn.Module):
continue
if self.config.tie_word_embeddings and "lm_head.weight" in name:
continue
if (
name == "model.embed_tokens.weight"
and self.config.tie_word_embeddings
and self.pp_group.world_size > 1
):
if self.pp_group.is_last_rank:
name = "lm_head.weight"
elif not self.pp_group.is_first_rank:
continue
# Handle FP8 kv-scale remapping
if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
+12 -1
View File
@@ -786,6 +786,7 @@ class ExaoneMoEForCausalLM(nn.Module):
if name.startswith("model.vision_tower") and name not in params_dict:
continue
is_expert_weight = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -807,7 +808,11 @@ class ExaoneMoEForCausalLM(nn.Module):
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
is_expert_weight = True
name = name.replace(weight_name, param_name)
if name not in params_dict:
# The expert's layer lives on another pipeline rank.
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
@@ -819,6 +824,8 @@ class ExaoneMoEForCausalLM(nn.Module):
)
break
else:
if is_expert_weight:
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
@@ -859,4 +866,8 @@ class ExaoneMoEForCausalLM(nn.Module):
self.model.layers_to_capture = [val + 1 for val in layer_ids]
EntryClass = ExaoneMoEForCausalLM
class ExaoneMoeForCausalLM(ExaoneMoEForCausalLM):
pass
EntryClass = [ExaoneMoEForCausalLM, ExaoneMoeForCausalLM]