[Fix] EXAONE and ERNIE 4.5 VL MoE architecture, backend and PP issues (#42303)
This commit is contained in:
@@ -636,7 +636,11 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
# The prefill attention backend default + validation moved to the
|
||||
# override registry (arg_groups/overrides.py: _moss_vl_overrides).
|
||||
pass
|
||||
elif model_arch in ["Exaone4ForCausalLM", "ExaoneMoEForCausalLM"]:
|
||||
elif model_arch in [
|
||||
"Exaone4ForCausalLM",
|
||||
"ExaoneMoEForCausalLM",
|
||||
"ExaoneMoeForCausalLM",
|
||||
]:
|
||||
if hf_config.sliding_window_pattern is not None:
|
||||
# disable_hybrid_swa_memory moved to the override registry
|
||||
# (arg_groups/overrides.py: _exaone_overrides).
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.arg_groups.model_overrides import bailing_moe_v3 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import deepseek_v2 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import deepseek_v4 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import ernie45_vl # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import exaone # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import falcon_h1 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import gemma2_gemma3 # noqa: F401
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Config-time override declarations for ernie45_vl."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
|
||||
|
||||
@_register_for("Ernie4_5_VLMoeForConditionalGeneration")
|
||||
def _ernie45_vl_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
if attn_dp_enabled_of(resolving_view(server_args)):
|
||||
raise ValueError(
|
||||
"ERNIE 4.5 VL MoE does not support DP attention: its attention "
|
||||
"heads are split over the full TP group."
|
||||
)
|
||||
return {}
|
||||
@@ -1,20 +1,30 @@
|
||||
"""Config-time override declarations for exaone."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from typing import Any, Dict
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM")
|
||||
@_register_for("Exaone4ForCausalLM", "ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
|
||||
def _exaone_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
overrides: Dict[str, Any] = {}
|
||||
if hf_config.sliding_window_pattern is not None:
|
||||
logger.warning(
|
||||
f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported."
|
||||
)
|
||||
return {"disable_hybrid_swa_memory": True}
|
||||
return {}
|
||||
overrides["disable_hybrid_swa_memory"] = True
|
||||
if resolving_view(server_args).attention_backend is None:
|
||||
if get_platform().is_cuda and get_platform().is_sm100:
|
||||
overrides["attention_backend"] = "trtllm_mha"
|
||||
elif get_platform().is_cuda and get_platform().device_sm >= 80:
|
||||
overrides["attention_backend"] = "fa3"
|
||||
else:
|
||||
overrides["attention_backend"] = "triton"
|
||||
return overrides
|
||||
|
||||
@@ -1030,7 +1030,10 @@ class ModelConfig:
|
||||
self.hf_config.num_nextn_predict_layers = 1
|
||||
self.hf_text_config.num_nextn_predict_layers = 1
|
||||
|
||||
if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM":
|
||||
if is_draft_model and self.hf_config.architectures[0] in (
|
||||
"ExaoneMoEForCausalLM",
|
||||
"ExaoneMoeForCausalLM",
|
||||
):
|
||||
self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
|
||||
self.hf_config.num_nextn_predict_layers = 1
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -479,7 +478,6 @@ class Ernie4_5_VLMoeModel(nn.Module):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
enable_tp=not is_dp_attention_enabled(),
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -435,7 +435,7 @@ class Exaone4ForCausalLM(nn.Module):
|
||||
self.model = self._init_model(config, quant_config, add_prefix("model", prefix))
|
||||
# Exaone-4.0 32B set tie_word_embeddins to False
|
||||
# Exaone-4.0 1.2B set tie_word_embeddins to True
|
||||
if config.tie_word_embeddings:
|
||||
if config.tie_word_embeddings and self.pp_group.world_size == 1:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
else:
|
||||
self.lm_head = ParallelLMHead(
|
||||
@@ -575,6 +575,15 @@ class Exaone4ForCausalLM(nn.Module):
|
||||
continue
|
||||
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
||||
continue
|
||||
if (
|
||||
name == "model.embed_tokens.weight"
|
||||
and self.config.tie_word_embeddings
|
||||
and self.pp_group.world_size > 1
|
||||
):
|
||||
if self.pp_group.is_last_rank:
|
||||
name = "lm_head.weight"
|
||||
elif not self.pp_group.is_first_rank:
|
||||
continue
|
||||
# Handle FP8 kv-scale remapping
|
||||
if "scale" in name:
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
|
||||
@@ -786,6 +786,7 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
if name.startswith("model.vision_tower") and name not in params_dict:
|
||||
continue
|
||||
|
||||
is_expert_weight = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
@@ -807,7 +808,11 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
# The expert's layer lives on another pipeline rank.
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
@@ -819,6 +824,8 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
)
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
@@ -859,4 +866,8 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
||||
|
||||
|
||||
EntryClass = ExaoneMoEForCausalLM
|
||||
class ExaoneMoeForCausalLM(ExaoneMoEForCausalLM):
|
||||
pass
|
||||
|
||||
|
||||
EntryClass = [ExaoneMoEForCausalLM, ExaoneMoeForCausalLM]
|
||||
|
||||
Reference in New Issue
Block a user