[Refactor] Pass each layer stack's output through a communicator exit (#41081)
This commit is contained in:
@@ -943,6 +943,17 @@ class LayerCommunicator:
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch)
|
||||
|
||||
def finish_layer_stack(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers."""
|
||||
return complete_deferred_allreduce(hidden_states), residual
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
return False
|
||||
|
||||
@@ -849,8 +849,11 @@ class BailingMoEModel(nn.Module):
|
||||
else None
|
||||
),
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -21,7 +21,6 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -1011,8 +1010,11 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -183,6 +183,9 @@ class BailingMoEModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
|
||||
@@ -1393,8 +1393,11 @@ class BailingMoELinearModel(nn.Module):
|
||||
else:
|
||||
dspark_aux_hidden_states.append(hidden_states + residual)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -226,6 +226,11 @@ class DeepseekModelNextN(nn.Module):
|
||||
zero_allocator,
|
||||
prev_topk_indices=index_topk_share.topk_indices,
|
||||
)
|
||||
hidden_states, residual = (
|
||||
self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
|
||||
@@ -70,7 +70,6 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
@@ -3109,8 +3108,11 @@ class DeepseekV2Model(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
proxy_tensors = {
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
|
||||
@@ -60,7 +60,6 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1823,8 +1822,11 @@ class Dots3Model(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -118,6 +118,9 @@ class Dot3NoteModelNextN(nn.Module):
|
||||
hidden_states, residual = head.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
)
|
||||
hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
|
||||
@@ -1139,8 +1139,11 @@ class Glm4MoeModel(nn.Module):
|
||||
].layer_scatter_modes.layer_output_mode,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -838,8 +838,11 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -120,6 +120,9 @@ class Glm4MoeLiteModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
|
||||
@@ -106,6 +106,9 @@ class Glm4MoeModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
|
||||
@@ -1216,8 +1216,11 @@ class Glm5NextModel(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
if self.config.mhc:
|
||||
return PPProxyTensors({"hidden_states": hidden_states})
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -737,8 +737,11 @@ class GptOssModel(nn.Module):
|
||||
if residual is not None
|
||||
else hidden_states
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -606,8 +606,11 @@ class LagunaModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1093,6 +1092,11 @@ class MiMoV2Model(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
# A draft targeting the final layer ("after layer
|
||||
# num_hidden_layers-1") maps to capture index num_hidden_layers,
|
||||
# past the layer loop; capture the pre-norm output here instead.
|
||||
@@ -1106,7 +1110,6 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
@@ -1186,8 +1185,11 @@ class MiniMaxM2Model(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1539,8 +1538,11 @@ class MiniMaxM3Model(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -34,7 +34,6 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import complete_deferred_allreduce
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
@@ -831,17 +830,15 @@ class NemotronHModel(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
if residual is not None and getattr(
|
||||
hidden_states, "_sglang_needs_allreduce_fusion", False
|
||||
):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
aux_hidden_states.append(
|
||||
self._capture_hidden_states(hidden_states, residual, self.end_layer)
|
||||
)
|
||||
|
||||
@@ -111,6 +111,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
|
||||
)
|
||||
|
||||
if self.has_end_norm:
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = None
|
||||
@@ -193,6 +196,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
)
|
||||
|
||||
if self.has_end_norm:
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = None
|
||||
|
||||
@@ -45,7 +45,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -1228,8 +1227,11 @@ class Qwen2MoeModel(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -1943,9 +1943,13 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
# Return intermediate tensors for pipeline parallelism
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -949,6 +949,10 @@ class Qwen3NextModel(nn.Module):
|
||||
),
|
||||
)
|
||||
|
||||
last_layer = self.layers[-1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -152,8 +152,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
else None
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1171,8 +1170,11 @@ class SarvamMLAModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -19,7 +19,6 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -490,8 +489,11 @@ class SDARMoeModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -738,8 +737,11 @@ class Step3p5Model(nn.Module):
|
||||
residual,
|
||||
)
|
||||
# break
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -115,6 +115,9 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
|
||||
Reference in New Issue
Block a user