[Refactor] Build the ERNIE 4.5 VL MoE and EXAONE MoE decoders from stage boundaries (#42304)

This commit is contained in:
Cheng Wan
2026-10-03 02:18:56 -07:00
committed by GitHub
parent f89ead192f
commit 2fa17a6467
2 changed files with 126 additions and 100 deletions
+66 -58
View File
@@ -16,15 +16,18 @@
import logging
from itertools import islice
from typing import Any, Dict, Optional, Tuple, Union
from typing import Any, Dict, Optional, Union
import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -107,6 +110,7 @@ class Ernie4_5_VLMoeAttention(nn.Module):
hidden_size,
bias=bias,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
@@ -348,12 +352,28 @@ class Ernie4_5_VLMoeMoE(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(orig_shape)
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
min_moe_layer_start_index = min(config.moe_layer_start_index)
moe_layer_end_index = getattr(
config,
"moe_layer_end_index",
[config.num_hidden_layers - 1, config.num_hidden_layers - 1],
)
max_moe_layer_end_index = max(moe_layer_end_index)
assert min_moe_layer_start_index <= max_moe_layer_end_index
moe_layer_interval = getattr(config, "moe_layer_interval", 1)
use_moe = getattr(config, "use_moe", max(config.moe_num_experts) > 0)
return (
use_moe
and ((layer_id + 1) % moe_layer_interval == 0)
and layer_id >= min_moe_layer_start_index
and layer_id <= max_moe_layer_end_index
)
class Ernie4_5_VLMoeDecoderLayer(nn.Module):
"""A single transformer layer.
@@ -391,26 +411,9 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
)
# MoE
moe_layer_start_index = config.moe_layer_start_index
min_moe_layer_start_index = min(moe_layer_start_index)
moe_layer_end_index = getattr(
config,
"moe_layer_end_index",
[config.num_hidden_layers - 1, config.num_hidden_layers - 1],
)
max_moe_layer_end_index = max(moe_layer_end_index)
assert min_moe_layer_start_index <= max_moe_layer_end_index
moe_num_experts = config.moe_num_experts
max_moe_num_experts = max(moe_num_experts)
moe_layer_interval = getattr(config, "moe_layer_interval", 1)
use_moe = getattr(config, "use_moe", max_moe_num_experts > 0)
# MLP
if (
use_moe
and ((layer_id + 1) % moe_layer_interval == 0)
and layer_id >= min_moe_layer_start_index
and layer_id <= max_moe_layer_end_index
):
sparse = _is_moe_layer(config, layer_id)
if sparse:
self.mlp = Ernie4_5_VLMoeMoE(
config=config,
layer_id=layer_id,
@@ -423,6 +426,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
allow_fused_down=False,
prefix=add_prefix("mlp", prefix),
)
@@ -430,22 +435,34 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
config, layer_id + 1
)
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=_is_moe_layer(config, layer_id - 1),
next_layer_sparse=sparse,
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
visual_token_mask: torch.Tensor | None,
**kwargs: object,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
@@ -453,13 +470,14 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
else:
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
else:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
# only used as text backbone for ernie4.5 vl
@@ -517,33 +535,23 @@ class Ernie4_5_VLMoeModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
visual_token_mask,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
return residual_batch.to_pp(hidden_states, forward_batch)
return residual_batch.final_norm(
hidden_states, forward_batch, self.norm, skip_empty=True
)
+60 -42
View File
@@ -28,9 +28,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -39,10 +46,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
reduce_moe_output,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -286,7 +290,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -358,6 +361,7 @@ class ExaoneMoEAttention(nn.Module):
hidden_size,
bias=bias,
quant_config=o_quant_config,
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
@@ -478,25 +482,44 @@ class ExaoneMoEDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("mlp", prefix),
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
is_moe_layer = config.is_moe_layer
num_layers = config.num_hidden_layers
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=is_moe_layer[layer_id],
next_layer_sparse=layer_id + 1 < num_layers
and is_moe_layer[layer_id + 1],
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_moe_layer[layer_id - 1],
next_layer_sparse=is_moe_layer[layer_id],
)
if layer_id != 0
else None,
terminal=layer_id == num_layers - 1,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture=capture_output
)
# Self Attention
hidden_states = self.self_attn(
@@ -505,11 +528,12 @@ class ExaoneMoEDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
# Fully Connected
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class ExaoneMoEModel(nn.Module):
@@ -570,34 +594,29 @@ class ExaoneMoEModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
aux_hidden_states.append(hidden_states + residual)
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
hidden_states = self.layers[i](
positions,
hidden_states,
forward_batch,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm, skip_empty=True
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -692,6 +711,7 @@ class ExaoneMoEForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -699,19 +719,17 @@ class ExaoneMoEForCausalLM(nn.Module):
# decoder layer
for i in range(start, end):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
forward_batch.hidden_states = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
)
if end == self.model.config.num_hidden_layers:
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
forward_batch.hidden_states = residual_batch.final_norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch