[Refactor] Pass each layer stack's output through a communicator exit (#41081)

This commit is contained in:
Cheng Wan
2026-09-24 12:15:07 -07:00
committed by GitHub
parent 07a3735e94
commit 5df1667077
33 changed files with 470 additions and 210 deletions
+11
View File
@@ -943,6 +943,17 @@ class LayerCommunicator:
result as a context manager around the FFN call, then call ``finish``."""
return FfnExit(self, forward_batch)
def finish_layer_stack(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers."""
return complete_deferred_allreduce(hidden_states), residual
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
if not self.allow_reduce_scatter:
return False
+4 -1
View File
@@ -849,8 +849,11 @@ class BailingMoEModel(nn.Module):
else None
),
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
@@ -21,7 +21,6 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -1011,8 +1010,11 @@ class BailingMoELinearModel(nn.Module):
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
@@ -183,6 +183,9 @@ class BailingMoEModelNextN(nn.Module):
positions, hidden_states, forward_batch, residual
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
+4 -1
View File
@@ -1393,8 +1393,11 @@ class BailingMoELinearModel(nn.Module):
else:
dspark_aux_hidden_states.append(hidden_states + residual)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
@@ -226,6 +226,11 @@ class DeepseekModelNextN(nn.Module):
zero_allocator,
prev_topk_indices=index_topk_share.topk_indices,
)
hidden_states, residual = (
self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
+4 -2
View File
@@ -70,7 +70,6 @@ from sglang.srt.layers.aux_hidden_states import (
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
@@ -3109,8 +3108,11 @@ class DeepseekV2Model(nn.Module):
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
proxy_tensors = {
"hidden_states": hidden_states,
"residual": residual,
@@ -60,7 +60,6 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1823,8 +1822,11 @@ class Dots3Model(nn.Module):
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
@@ -118,6 +118,9 @@ class Dot3NoteModelNextN(nn.Module):
hidden_states, residual = head.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
)
hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is None:
+4 -1
View File
@@ -1139,8 +1139,11 @@ class Glm4MoeModel(nn.Module):
].layer_scatter_modes.layer_output_mode,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+4 -1
View File
@@ -838,8 +838,11 @@ class Glm4MoeLiteModel(nn.Module):
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
@@ -120,6 +120,9 @@ class Glm4MoeLiteModelNextN(nn.Module):
positions, hidden_states, forward_batch, residual, zero_allocator
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
@@ -106,6 +106,9 @@ class Glm4MoeModelNextN(nn.Module):
positions, hidden_states, forward_batch, residual
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
+4 -1
View File
@@ -1216,8 +1216,11 @@ class Glm5NextModel(nn.Module):
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
if self.config.mhc:
return PPProxyTensors({"hidden_states": hidden_states})
return PPProxyTensors(
+4 -1
View File
@@ -737,8 +737,11 @@ class GptOssModel(nn.Module):
if residual is not None
else hidden_states
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+4 -1
View File
@@ -606,8 +606,11 @@ class LagunaModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+5 -2
View File
@@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1093,6 +1092,11 @@ class MiMoV2Model(nn.Module):
),
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
# A draft targeting the final layer ("after layer
# num_hidden_layers-1") maps to capture index num_hidden_layers,
# past the layer loop; capture the pre-norm output here instead.
@@ -1106,7 +1110,6 @@ class MiMoV2Model(nn.Module):
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+4 -2
View File
@@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
@@ -1186,8 +1185,11 @@ class MiniMaxM2Model(nn.Module):
),
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+4 -2
View File
@@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1539,8 +1538,11 @@ class MiniMaxM3Model(nn.Module):
),
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+4 -7
View File
@@ -34,7 +34,6 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import complete_deferred_allreduce
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
@@ -831,17 +830,15 @@ class NemotronHModel(nn.Module):
forward_batch=forward_batch,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if self.end_layer in self.layers_to_capture:
if residual is not None and getattr(
hidden_states, "_sglang_needs_allreduce_fusion", False
):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states._sglang_needs_allreduce_fusion = False
aux_hidden_states.append(
self._capture_hidden_states(hidden_states, residual, self.end_layer)
)
@@ -111,6 +111,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
)
if self.has_end_norm:
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states = hidden_states + residual
residual = None
@@ -193,6 +196,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
)
if self.has_end_norm:
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states = hidden_states + residual
residual = None
+4 -2
View File
@@ -45,7 +45,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -1228,8 +1227,11 @@ class Qwen2MoeModel(nn.Module):
),
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+5 -1
View File
@@ -1943,9 +1943,13 @@ class Qwen3_5ForCausalLM(nn.Module):
input_deepstack_embeds[:, sep : sep + self.hidden_size]
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
# Return intermediate tensors for pipeline parallelism
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+4
View File
@@ -949,6 +949,10 @@ class Qwen3NextModel(nn.Module):
),
)
last_layer = self.layers[-1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
+4 -1
View File
@@ -152,8 +152,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
else None
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+4 -2
View File
@@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1171,8 +1170,11 @@ class SarvamMLAModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+4 -2
View File
@@ -19,7 +19,6 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -490,8 +489,11 @@ class SDARMoeModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+4 -2
View File
@@ -14,7 +14,6 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -738,8 +737,11 @@ class Step3p5Model(nn.Module):
residual,
)
# break
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+3
View File
@@ -115,6 +115,9 @@ class Step3p5AMultiTokenPredictor(nn.Module):
forward_batch=forward_batch,
residual=None,
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
# if forward_batch.return_hidden_states_before_norm: