From f89ead192f3cf7a541be88518691fd6a95e9ff3e Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:18:33 -0700 Subject: [PATCH] [Fix] EXAONE and ERNIE 4.5 VL MoE architecture, backend and PP issues (#42303) --- python/sglang/srt/arg_groups/model_hook.py | 6 +++++- .../arg_groups/model_overrides/__init__.py | 1 + .../arg_groups/model_overrides/ernie45_vl.py | 19 +++++++++++++++++++ .../srt/arg_groups/model_overrides/exaone.py | 18 ++++++++++++++---- python/sglang/srt/configs/model_config.py | 5 ++++- python/sglang/srt/models/ernie45_moe_vl.py | 2 -- python/sglang/srt/models/exaone4.py | 11 ++++++++++- python/sglang/srt/models/exaone_moe.py | 13 ++++++++++++- 8 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 python/sglang/srt/arg_groups/model_overrides/ernie45_vl.py diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 1fe46d0467..b75a0a51f4 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -636,7 +636,11 @@ def handle_model_specific_adjustments(server_args: Any): # The prefill attention backend default + validation moved to the # override registry (arg_groups/overrides.py: _moss_vl_overrides). pass - elif model_arch in ["Exaone4ForCausalLM", "ExaoneMoEForCausalLM"]: + elif model_arch in [ + "Exaone4ForCausalLM", + "ExaoneMoEForCausalLM", + "ExaoneMoeForCausalLM", + ]: if hf_config.sliding_window_pattern is not None: # disable_hybrid_swa_memory moved to the override registry # (arg_groups/overrides.py: _exaone_overrides). diff --git a/python/sglang/srt/arg_groups/model_overrides/__init__.py b/python/sglang/srt/arg_groups/model_overrides/__init__.py index 4a64a30b31..b0be0a9556 100644 --- a/python/sglang/srt/arg_groups/model_overrides/__init__.py +++ b/python/sglang/srt/arg_groups/model_overrides/__init__.py @@ -12,6 +12,7 @@ from sglang.srt.arg_groups.model_overrides import bailing_moe_v3 # noqa: F401 from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401 from sglang.srt.arg_groups.model_overrides import deepseek_v2 # noqa: F401 from sglang.srt.arg_groups.model_overrides import deepseek_v4 # noqa: F401 +from sglang.srt.arg_groups.model_overrides import ernie45_vl # noqa: F401 from sglang.srt.arg_groups.model_overrides import exaone # noqa: F401 from sglang.srt.arg_groups.model_overrides import falcon_h1 # noqa: F401 from sglang.srt.arg_groups.model_overrides import gemma2_gemma3 # noqa: F401 diff --git a/python/sglang/srt/arg_groups/model_overrides/ernie45_vl.py b/python/sglang/srt/arg_groups/model_overrides/ernie45_vl.py new file mode 100644 index 0000000000..8db1376796 --- /dev/null +++ b/python/sglang/srt/arg_groups/model_overrides/ernie45_vl.py @@ -0,0 +1,19 @@ +"""Config-time override declarations for ernie45_vl.""" + +from typing import Any + +from sglang.srt.arg_groups.model_override_base import ( + _register_for, + resolving_view, +) +from sglang.srt.runtime_context import attn_dp_enabled_of + + +@_register_for("Ernie4_5_VLMoeForConditionalGeneration") +def _ernie45_vl_overrides(server_args: Any, hf_config: Any) -> dict: + if attn_dp_enabled_of(resolving_view(server_args)): + raise ValueError( + "ERNIE 4.5 VL MoE does not support DP attention: its attention " + "heads are split over the full TP group." + ) + return {} diff --git a/python/sglang/srt/arg_groups/model_overrides/exaone.py b/python/sglang/srt/arg_groups/model_overrides/exaone.py index 7d0615f2c6..248cad09fe 100644 --- a/python/sglang/srt/arg_groups/model_overrides/exaone.py +++ b/python/sglang/srt/arg_groups/model_overrides/exaone.py @@ -1,20 +1,30 @@ """Config-time override declarations for exaone.""" import logging -from typing import Any +from typing import Any, Dict from sglang.srt.arg_groups.model_override_base import ( _register_for, + resolving_view, ) +from sglang.srt.runtime_context import get_platform logger = logging.getLogger(__name__) -@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM") +@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM") def _exaone_overrides(server_args: Any, hf_config: Any) -> dict: + overrides: Dict[str, Any] = {} if hf_config.sliding_window_pattern is not None: logger.warning( f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported." ) - return {"disable_hybrid_swa_memory": True} - return {} + overrides["disable_hybrid_swa_memory"] = True + if resolving_view(server_args).attention_backend is None: + if get_platform().is_cuda and get_platform().is_sm100: + overrides["attention_backend"] = "trtllm_mha" + elif get_platform().is_cuda and get_platform().device_sm >= 80: + overrides["attention_backend"] = "fa3" + else: + overrides["attention_backend"] = "triton" + return overrides diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index fd97b595b6..c76b8dc30e 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1030,7 +1030,10 @@ class ModelConfig: self.hf_config.num_nextn_predict_layers = 1 self.hf_text_config.num_nextn_predict_layers = 1 - if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM": + if is_draft_model and self.hf_config.architectures[0] in ( + "ExaoneMoEForCausalLM", + "ExaoneMoeForCausalLM", + ): self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP" self.hf_config.num_nextn_predict_layers = 1 diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index 362ed49bd1..ddfe41f0db 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -25,7 +25,6 @@ from transformers import PretrainedConfig from sglang.srt.distributed import ( tensor_model_parallel_all_reduce, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -479,7 +478,6 @@ class Ernie4_5_VLMoeModel(nn.Module): self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, - enable_tp=not is_dp_attention_enabled(), prefix=add_prefix("embed_tokens", prefix), ) else: diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index 78edb9969a..2b013ff4de 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -435,7 +435,7 @@ class Exaone4ForCausalLM(nn.Module): self.model = self._init_model(config, quant_config, add_prefix("model", prefix)) # Exaone-4.0 32B set tie_word_embeddins to False # Exaone-4.0 1.2B set tie_word_embeddins to True - if config.tie_word_embeddings: + if config.tie_word_embeddings and self.pp_group.world_size == 1: self.lm_head = self.model.embed_tokens else: self.lm_head = ParallelLMHead( @@ -575,6 +575,15 @@ class Exaone4ForCausalLM(nn.Module): continue if self.config.tie_word_embeddings and "lm_head.weight" in name: continue + if ( + name == "model.embed_tokens.weight" + and self.config.tie_word_embeddings + and self.pp_group.world_size > 1 + ): + if self.pp_group.is_last_rank: + name = "lm_head.weight" + elif not self.pp_group.is_first_rank: + continue # Handle FP8 kv-scale remapping if "scale" in name: name = maybe_remap_kv_scale_name(name, params_dict) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 125a96faa2..19b4fb4964 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -786,6 +786,7 @@ class ExaoneMoEForCausalLM(nn.Module): if name.startswith("model.vision_tower") and name not in params_dict: continue + is_expert_weight = False for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue @@ -807,7 +808,11 @@ class ExaoneMoEForCausalLM(nn.Module): param_name, weight_name, expert_id, shard_id = mapping if weight_name not in name: continue + is_expert_weight = True name = name.replace(weight_name, param_name) + if name not in params_dict: + # The expert's layer lives on another pipeline rank. + continue param = params_dict[name] weight_loader = param.weight_loader weight_loader( @@ -819,6 +824,8 @@ class ExaoneMoEForCausalLM(nn.Module): ) break else: + if is_expert_weight: + continue # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue @@ -859,4 +866,8 @@ class ExaoneMoEForCausalLM(nn.Module): self.model.layers_to_capture = [val + 1 for val in layer_ids] -EntryClass = ExaoneMoEForCausalLM +class ExaoneMoeForCausalLM(ExaoneMoEForCausalLM): + pass + + +EntryClass = [ExaoneMoEForCausalLM, ExaoneMoeForCausalLM]