[Refactor] Build the ERNIE 4.5 VL MoE and EXAONE MoE decoders from stage boundaries (#42304)
This commit is contained in:
@@ -16,15 +16,18 @@
|
||||
|
||||
import logging
|
||||
from itertools import islice
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -107,6 +110,7 @@ class Ernie4_5_VLMoeAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -348,12 +352,28 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.tp_size > 1:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(orig_shape)
|
||||
|
||||
|
||||
def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool:
|
||||
min_moe_layer_start_index = min(config.moe_layer_start_index)
|
||||
moe_layer_end_index = getattr(
|
||||
config,
|
||||
"moe_layer_end_index",
|
||||
[config.num_hidden_layers - 1, config.num_hidden_layers - 1],
|
||||
)
|
||||
max_moe_layer_end_index = max(moe_layer_end_index)
|
||||
assert min_moe_layer_start_index <= max_moe_layer_end_index
|
||||
moe_layer_interval = getattr(config, "moe_layer_interval", 1)
|
||||
use_moe = getattr(config, "use_moe", max(config.moe_num_experts) > 0)
|
||||
return (
|
||||
use_moe
|
||||
and ((layer_id + 1) % moe_layer_interval == 0)
|
||||
and layer_id >= min_moe_layer_start_index
|
||||
and layer_id <= max_moe_layer_end_index
|
||||
)
|
||||
|
||||
|
||||
class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
"""A single transformer layer.
|
||||
|
||||
@@ -391,26 +411,9 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# MoE
|
||||
moe_layer_start_index = config.moe_layer_start_index
|
||||
min_moe_layer_start_index = min(moe_layer_start_index)
|
||||
moe_layer_end_index = getattr(
|
||||
config,
|
||||
"moe_layer_end_index",
|
||||
[config.num_hidden_layers - 1, config.num_hidden_layers - 1],
|
||||
)
|
||||
max_moe_layer_end_index = max(moe_layer_end_index)
|
||||
assert min_moe_layer_start_index <= max_moe_layer_end_index
|
||||
moe_num_experts = config.moe_num_experts
|
||||
max_moe_num_experts = max(moe_num_experts)
|
||||
moe_layer_interval = getattr(config, "moe_layer_interval", 1)
|
||||
use_moe = getattr(config, "use_moe", max_moe_num_experts > 0)
|
||||
# MLP
|
||||
if (
|
||||
use_moe
|
||||
and ((layer_id + 1) % moe_layer_interval == 0)
|
||||
and layer_id >= min_moe_layer_start_index
|
||||
and layer_id <= max_moe_layer_end_index
|
||||
):
|
||||
sparse = _is_moe_layer(config, layer_id)
|
||||
if sparse:
|
||||
self.mlp = Ernie4_5_VLMoeMoE(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
@@ -423,6 +426,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
@@ -430,22 +435,34 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer(
|
||||
config, layer_id + 1
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=sparse, next_layer_sparse=next_sparse),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=_is_moe_layer(config, layer_id - 1),
|
||||
next_layer_sparse=sparse,
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
visual_token_mask: torch.Tensor | None,
|
||||
**kwargs: object,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
@@ -453,13 +470,14 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
|
||||
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Ernie4_5_VLMoeMoE):
|
||||
hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
# only used as text backbone for ernie4.5 vl
|
||||
@@ -517,33 +535,23 @@ class Ernie4_5_VLMoeModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
visual_token_mask,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
return hidden_states
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
return residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self.norm, skip_empty=True
|
||||
)
|
||||
|
||||
@@ -28,9 +28,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -39,10 +46,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -286,7 +290,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
@@ -358,6 +361,7 @@ class ExaoneMoEAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=bias,
|
||||
quant_config=o_quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
@@ -478,25 +482,44 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
is_moe_layer = config.is_moe_layer
|
||||
num_layers = config.num_hidden_layers
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer[layer_id],
|
||||
next_layer_sparse=layer_id + 1 < num_layers
|
||||
and is_moe_layer[layer_id + 1],
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_moe_layer[layer_id - 1],
|
||||
next_layer_sparse=is_moe_layer[layer_id],
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == num_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture=capture_output
|
||||
)
|
||||
|
||||
# Self Attention
|
||||
hidden_states = self.self_attn(
|
||||
@@ -505,11 +528,12 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
# Fully Connected
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class ExaoneMoEModel(nn.Module):
|
||||
@@ -570,34 +594,29 @@ class ExaoneMoEModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
hidden_states = self.layers[i](
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self.norm, skip_empty=True
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -692,6 +711,7 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -699,19 +719,17 @@ class ExaoneMoEForCausalLM(nn.Module):
|
||||
# decoder layer
|
||||
for i in range(start, end):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
forward_batch.hidden_states = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
forward_batch.hidden_states = residual_batch.final_norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
|
||||
Reference in New Issue
Block a user