diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index ddfe41f0db..9261e7240c 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -16,15 +16,18 @@ import logging from itertools import islice -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, ) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -107,6 +110,7 @@ class Ernie4_5_VLMoeAttention(nn.Module): hidden_size, bias=bias, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -348,12 +352,28 @@ class Ernie4_5_VLMoeMoE(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1: - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) - return final_hidden_states.view(orig_shape) +def _is_moe_layer(config: PretrainedConfig, layer_id: int) -> bool: + min_moe_layer_start_index = min(config.moe_layer_start_index) + moe_layer_end_index = getattr( + config, + "moe_layer_end_index", + [config.num_hidden_layers - 1, config.num_hidden_layers - 1], + ) + max_moe_layer_end_index = max(moe_layer_end_index) + assert min_moe_layer_start_index <= max_moe_layer_end_index + moe_layer_interval = getattr(config, "moe_layer_interval", 1) + use_moe = getattr(config, "use_moe", max(config.moe_num_experts) > 0) + return ( + use_moe + and ((layer_id + 1) % moe_layer_interval == 0) + and layer_id >= min_moe_layer_start_index + and layer_id <= max_moe_layer_end_index + ) + + class Ernie4_5_VLMoeDecoderLayer(nn.Module): """A single transformer layer. @@ -391,26 +411,9 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): ) # MoE - moe_layer_start_index = config.moe_layer_start_index - min_moe_layer_start_index = min(moe_layer_start_index) - moe_layer_end_index = getattr( - config, - "moe_layer_end_index", - [config.num_hidden_layers - 1, config.num_hidden_layers - 1], - ) - max_moe_layer_end_index = max(moe_layer_end_index) - assert min_moe_layer_start_index <= max_moe_layer_end_index - moe_num_experts = config.moe_num_experts - max_moe_num_experts = max(moe_num_experts) - moe_layer_interval = getattr(config, "moe_layer_interval", 1) - use_moe = getattr(config, "use_moe", max_moe_num_experts > 0) # MLP - if ( - use_moe - and ((layer_id + 1) % moe_layer_interval == 0) - and layer_id >= min_moe_layer_start_index - and layer_id <= max_moe_layer_end_index - ): + sparse = _is_moe_layer(config, layer_id) + if sparse: self.mlp = Ernie4_5_VLMoeMoE( config=config, layer_id=layer_id, @@ -423,6 +426,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, + reduce_results=False, + allow_fused_down=False, prefix=add_prefix("mlp", prefix), ) @@ -430,22 +435,34 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + next_sparse = layer_id + 1 < config.num_hidden_layers and _is_moe_layer( + config, layer_id + 1 + ) + self.attn_boundary, self.ffn_boundary = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn(sparse=sparse, next_layer_sparse=next_sparse), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=_is_moe_layer(config, layer_id - 1), + next_layer_sparse=sparse, + ) + if layer_id != 0 + else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], visual_token_mask: torch.Tensor | None, **kwargs: object, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, @@ -453,13 +470,14 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module): ) # Fully Connected - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - if isinstance(self.mlp, Ernie4_5_VLMoeMoE): - hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs) - else: - hidden_states = self.mlp(hidden_states) - - return hidden_states, residual + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + if isinstance(self.mlp, Ernie4_5_VLMoeMoE): + hidden_states = self.mlp(hidden_states, visual_token_mask, **kwargs) + else: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) # only used as text backbone for ernie4.5 vl @@ -517,33 +535,23 @@ class Ernie4_5_VLMoeModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) for layer in islice(self.layers, self.start_layer, self.end_layer): - hidden_states, residual = layer( + hidden_states = layer( positions, hidden_states, forward_batch, - residual, visual_token_mask, ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) - - if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) - - return hidden_states + return residual_batch.to_pp(hidden_states, forward_batch) + return residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 19b4fb4964..8462294b76 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -28,9 +28,16 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -39,10 +46,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput -from sglang.srt.layers.moe import ( - get_moe_a2a_backend, - reduce_moe_output, -) +from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -286,7 +290,6 @@ class ExaoneMoESparseMoEBlock(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) @@ -358,6 +361,7 @@ class ExaoneMoEAttention(nn.Module): hidden_size, bias=bias, quant_config=o_quant_config, + reduce_results=False, prefix=add_prefix("o_proj", prefix), tp_rank=attn_tp_rank, tp_size=attn_tp_size, @@ -478,25 +482,44 @@ class ExaoneMoEDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("mlp", prefix), ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + is_moe_layer = config.is_moe_layer + num_layers = config.num_hidden_layers + self.attn_boundary, self.ffn_boundary = make_stages( + (declare_attn(), self.input_layernorm), + ( + declare_ffn( + sparse=is_moe_layer[layer_id], + next_layer_sparse=layer_id + 1 < num_layers + and is_moe_layer[layer_id + 1], + ), + self.post_attention_layernorm, + ), + previous=declare_ffn( + sparse=is_moe_layer[layer_id - 1], + next_layer_sparse=is_moe_layer[layer_id], + ) + if layer_id != 0 + else None, + terminal=layer_id == num_layers - 1, + ) def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + capture_output=None, + ) -> torch.Tensor: + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture=capture_output + ) # Self Attention hidden_states = self.self_attn( @@ -505,11 +528,12 @@ class ExaoneMoEDecoderLayer(nn.Module): forward_batch=forward_batch, ) - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) # Fully Connected - hidden_states = self.mlp(hidden_states) - - return hidden_states, residual + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class ExaoneMoEModel(nn.Module): @@ -570,34 +594,29 @@ class ExaoneMoEModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): - if i in self.layers_to_capture: - aux_hidden_states.append(hidden_states + residual) - layer = self.layers[i] - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual + hidden_states = self.layers[i]( + positions, + hidden_states, + forward_batch, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) - else: - if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) if len(aux_hidden_states) == 0: return hidden_states @@ -692,6 +711,7 @@ class ExaoneMoEForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -699,19 +719,17 @@ class ExaoneMoEForCausalLM(nn.Module): # decoder layer for i in range(start, end): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( + forward_batch.hidden_states = layer( positions, forward_batch.hidden_states, forward_batch, - forward_batch.residual, ) if end == self.model.config.num_hidden_layers: # norm - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual + forward_batch.hidden_states = residual_batch.final_norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) - forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( input_ids, forward_batch.hidden_states, self.lm_head, forward_batch