[Refactor] Centralize decoder output access (#41548)

This commit is contained in:
Cheng Wan
2026-09-28 20:14:04 -07:00
committed by GitHub
parent 0a8ca85565
commit 6f72cc3b64
52 changed files with 750 additions and 298 deletions
@@ -59,7 +59,6 @@ from sglang.srt.layers.communicator.layout import (
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
layer_input_buffer,
move_rows,
tp_reduce_scatter,
)
@@ -119,7 +118,6 @@ __all__ = [
"enable_moe_dense_fully_dp",
"get_attn_tp_context",
"input_scattered_layer_sides",
"layer_input_buffer",
"make_boundary",
"make_output_boundary",
"moe_cp_gathers_sparse_moe_input",
@@ -100,6 +100,14 @@ class StageOutput(msgspec.Struct, frozen=True):
update: StageUpdate = ADD
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class DecoderLayerSides(msgspec.Struct, frozen=True):
"""What a decoder layer declares about its attention, its FFN and the rows
between them; ``decoder_layer_edges`` turns it into the layer's
@@ -107,10 +115,8 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageInput
attention_output: StageOutput
ffn: StageInput
ffn_output: StageOutput
attention: StageDecl
ffn: StageDecl
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
@@ -122,14 +128,6 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
residual_joins_attention_sum: bool = False
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
@@ -167,7 +165,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
sides.input_rows,
group=sides.input_owes,
always_leaves=owes,
update=sides.ffn_output.update,
update=sides.ffn.output.update,
)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
@@ -179,19 +177,19 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
return DecoderLayerEdges(
into_attention=EdgeDecl(
produced=arrived,
need=sides.attention,
need=sides.attention.input,
residual=sides.input_rows,
residual_to=attention_rows,
),
into_ffn=EdgeDecl(
produced=sides.attention_output,
need=sides.ffn,
produced=sides.attention.output,
need=sides.ffn.input,
residual=attention_rows,
residual_to=sides.ffn_residual_rows,
residual_joins_sum=sides.residual_joins_attention_sum,
),
out_of_ffn=EdgeDecl(
produced=sides.ffn_output,
produced=sides.ffn.output,
need=StageInput(sides.output_rows),
residual=sides.ffn_residual_rows,
residual_to=sides.output_rows,
@@ -207,12 +205,14 @@ def with_residual(
replace = msgspec.structs.replace
return replace(
sides,
attention=replace(sides.attention, read=residual.attention_read),
attention_output=replace(
sides.attention_output, update=residual.attention_update
attention=StageDecl(
input=replace(sides.attention.input, read=residual.attention_read),
output=replace(sides.attention.output, update=residual.attention_update),
),
ffn=StageDecl(
input=replace(sides.ffn.input, read=residual.ffn_read),
output=replace(sides.ffn.output, update=residual.ffn_update),
),
ffn=replace(sides.ffn, read=residual.ffn_read),
ffn_output=replace(sides.ffn_output, update=residual.ffn_update),
)
@@ -270,14 +270,17 @@ def sequence_parallel_layer_sides(
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather the slices into their GEMM.
attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
# o_proj and down reduce-scatter out of theirs, whether fused or not.
attention_output=StageOutput(local),
ffn=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
# qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out.
attention=StageDecl(
input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
output=StageOutput(local),
),
ffn=StageDecl(
input=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
),
output=StageOutput(local),
),
ffn_output=StageOutput(local),
ffn_residual_rows=local,
output_rows=local,
)
@@ -311,19 +314,21 @@ def scattered_residual_layer_sides(
)
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
ffn_output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
ffn=StageDecl(
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
),
),
ffn_residual_rows=local,
output_rows=attention if is_last_layer else local,
@@ -347,18 +352,20 @@ def input_scattered_layer_sides(
)
return DecoderLayerSides(
input_rows=attention,
attention=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next layer's input completes the sum; the last layer's FFN does.
ffn_output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
ffn=StageDecl(
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next input completes the sum; the last FFN completes its own.
output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
),
),
ffn_residual_rows=attention,
output_rows=attention,
@@ -412,24 +419,26 @@ def decoder_layer_sides(
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
input_rows=local if previous_on_local_rows else attention,
attention=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
read=NORM_QUANT_READ,
),
# The output projection leaves the attention-TP sum to prepare_mlp.
output=StageOutput(
attention,
group=SumGroup.ATTN_TP if attention_tp else None,
always_leaves=attention_tp,
),
read=NORM_QUANT_READ,
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
attention,
group=SumGroup.ATTN_TP if attention_tp else None,
always_leaves=attention_tp,
),
ffn=StageInput(ffn),
ffn_output=(
StageOutput(ffn)
ffn=StageDecl(
input=StageInput(ffn),
output=StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
@@ -437,7 +446,7 @@ def decoder_layer_sides(
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
)
),
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=(
+51 -22
View File
@@ -68,11 +68,18 @@ from sglang.srt.layers.communicator.ops import (
move_rows,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.communicator.residual.access import (
add_to_output,
buffer,
finish_layer_stack,
fold,
from_pp,
snapshot,
)
from sglang.srt.layers.communicator.residual.add_norm import (
PLAIN_RESIDUAL,
aiter_all_reduce_fusion_enabled_for,
@@ -221,6 +228,11 @@ class LayerFusions(Protocol):
class LayerCommunicator:
add_to_output = staticmethod(add_to_output)
buffer = staticmethod(buffer)
fold = staticmethod(fold)
finish_layer_stack = staticmethod(finish_layer_stack)
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
# layout and try no fused kernel at the FFN exit.
_publish_lora_layout: bool = False
@@ -229,6 +241,7 @@ class LayerCommunicator:
_residual: LayerResidual = PLAIN_RESIDUAL
# The fused kernels a backend gives the layer, if any.
fusions: Optional[LayerFusions] = None
stage_edges = None
def __init__(
self,
@@ -534,14 +547,14 @@ class LayerCommunicator:
if self._residual.attention_update.adds_plainly:
scattered = input_scattered_layer_sides(
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
ffn_group=sides.ffn.output.group,
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
)
handoff = _hand_qkv_hook_its_input
else:
scattered = scattered_residual_layer_sides(
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
ffn_group=sides.ffn.output.group,
is_first_layer=self.layer_facts.is_first_layer,
is_last_layer=self.is_last_layer,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
@@ -566,6 +579,39 @@ class LayerCommunicator:
**kwargs,
)
def snapshot(self, hidden_states, residual, *, at_input: bool = False):
group = None
if residual is not None and self.stage_edges is not None:
edge = self.stage_edges[0 if at_input else 1]
if edge.produced.always_leaves:
group = _sum_group(edge.produced.group)
return snapshot(hidden_states, residual, group=group)
def capture_output(
self,
hidden_states,
residual,
*,
at_input: bool = False,
skip_empty: bool = False,
):
"""Complete the carried output on the main path, then snapshot it.
Keep this explicit completion distinct from a read-only snapshot: the
next prepare must not select a fusion for a sum already completed here."""
if skip_empty:
storage = buffer(hidden_states)
if storage is not None and storage.shape[0] == 0:
return hidden_states, None
hidden_states = reduce_output(hidden_states)
return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input)
def from_pp(self, tensors, *, allow_missing_residual: bool = False):
return from_pp(
tensors,
residual_in_hidden=self._residual.ffn_update.at_producer,
allow_missing_residual=allow_missing_residual,
)
def prepare_attn_and_capture_last_layer_outputs(
self,
hidden_states: torch.Tensor,
@@ -977,7 +1023,7 @@ class LayerCommunicator:
if self._declared is None:
raise NotImplementedError("a branch on a layer that is one stage")
sides = self._declared
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
return sides.ffn.input.layout, sides.ffn_residual_rows, sides.output_rows
def branch_input(
self,
@@ -1022,23 +1068,6 @@ class LayerCommunicator:
residual = move_rows(residual, rows, to, forward_batch)
return contribution + hidden_states, residual
def finish_layer_stack(
self,
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
*,
final_norm_takes_handoff: bool = False,
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers. A final norm
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
return hidden_states, residual
return reduce_output(hidden_states), residual
def _select_ffn_exit_fusions(
self,
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
@@ -1304,7 +1333,7 @@ class MHCLayerCommunicator(LayerCommunicator):
# MHC has not been run with an FFN input gathered over attention CP.
if (
TokenAxis.ATTN_CP
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
in sides.attention.output.layout.sharded - sides.ffn.input.layout.sharded
):
raise NotImplementedError(
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
@@ -102,15 +102,6 @@ def tp_reduce_scatter(
return output, residual
def layer_input_buffer(
hidden_states: Union[torch.Tensor, UnreducedOutput],
) -> torch.Tensor:
"""The tensor holding a layer's input, for reusing its memory without reading it."""
if isinstance(hidden_states, UnreducedOutput):
return hidden_states.partial
return hidden_states
class CommunicateSimpleFn:
@staticmethod
def _trivial(
@@ -0,0 +1,104 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Access to a layer output outside the next stage's prepare call."""
from typing import Optional, Tuple, Union
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
def buffer(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
) -> Optional[torch.Tensor]:
"""Storage that can be reused after prepare consumes the input. A finalize
handoff has no reusable layer-output tensor. This does not complete or read
the value held in the storage."""
if isinstance(hidden_states, UnreducedOutput):
return hidden_states.partial
if isinstance(hidden_states, HandoffOutput):
return None
return hidden_states
def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple:
"""Complete the output before adding an extra contribution exactly once.
Leave its residual update for the next prepare call."""
hidden_states = reduce_output(hidden_states)
hidden_states.add_(extra)
return hidden_states, residual
def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]:
"""Finish a plain layer output and fold its residual into a complete value.
The following stage receives it with no outstanding residual addition."""
hidden_states = reduce_output(hidden_states)
if residual is not None:
hidden_states = hidden_states + residual
return hidden_states, None
def finish_layer_stack(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
*,
final_norm_takes_handoff: bool = False,
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers. A final norm
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
return hidden_states, residual
return reduce_output(hidden_states), residual
def from_pp(
tensors: PPProxyTensors,
*,
residual_in_hidden: bool = False,
allow_missing_residual: bool = False,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Receive the layer-stack state without reducing a declared partial sum.
MHC carries its written streams in hidden_states; ordinary layers receive
a separate residual tensor."""
if residual_in_hidden:
residual = None
elif allow_missing_residual:
residual = tensors.tensors.get("residual")
else:
residual = tensors["residual"]
return tensors["hidden_states"], residual
def snapshot(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
*,
group: Optional[GroupCoordinator] = None,
) -> torch.Tensor:
"""Copy a complete output with its plain residual. A statically declared
sum is reduced on a copy; the main output and residual remain unchanged."""
if group is not None:
hidden_states = group.all_reduce(hidden_states.clone())
return hidden_states.clone() if residual is None else hidden_states + residual
+8 -6
View File
@@ -35,7 +35,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
enable_moe_dense_fully_dp,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -818,17 +817,20 @@ class BailingMoEModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states if residual is None else hidden_states + residual
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -983,8 +983,9 @@ class BailingMoELinearModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
total_num_layers = self.end_layer - self.start_layer
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
+9 -14
View File
@@ -32,8 +32,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
enable_moe_dense_fully_dp,
layer_input_buffer,
reduce_output,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
@@ -1344,8 +1342,9 @@ class BailingMoELinearModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
total_num_layers = self.end_layer - self.start_layer
device = hidden_states.device
@@ -1382,16 +1381,12 @@ class BailingMoELinearModel(nn.Module):
residual=residual,
zero_allocator=zero_allocator,
)
if (
capture_aux
and i in self.layers_to_capture
and layer_input_buffer(hidden_states).shape[0] != 0
):
hidden_states = reduce_output(hidden_states)
if residual is None:
dspark_aux_hidden_states.append(hidden_states)
else:
dspark_aux_hidden_states.append(hidden_states + residual)
if capture_aux and i in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, skip_empty=True
)
if snapshot is not None:
dspark_aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
+4 -4
View File
@@ -72,7 +72,6 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
maybe_prefetch_next_full_attention_kv,
@@ -2706,7 +2705,7 @@ class DeepseekV2DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
next_full_attention_layer_id: Optional[int] = None,
) -> torch.Tensor:
hidden_states_orig = layer_input_buffer(hidden_states)
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
@@ -3032,8 +3031,9 @@ class DeepseekV2Model(nn.Module):
initial_topk_indices = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
if not self.pp_group.is_first_rank:
@@ -1768,8 +1768,9 @@ class Dots3Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
+4
View File
@@ -432,6 +432,10 @@ class FalconH1Model(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.final_layernorm(hidden_states)
+7 -2
View File
@@ -561,8 +561,9 @@ class GigaChat35Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
total_num_layers = self.end_layer - self.start_layer
zero_allocator = BumpAllocator(
@@ -580,6 +581,10 @@ class GigaChat35Model(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
+6 -2
View File
@@ -115,10 +115,14 @@ class GigaChat35ModelNextN(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
hidden_states_before_norm = (
hidden_states if residual is None else hidden_states + residual
hidden_states_before_norm = self.decoder.layer_communicator.snapshot(
hidden_states, residual
)
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
+9 -5
View File
@@ -42,7 +42,6 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
enable_moe_dense_fully_dp,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
@@ -1088,8 +1087,9 @@ class Glm4MoeModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
@@ -1106,8 +1106,12 @@ class Glm4MoeModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(hidden_states + residual)
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
+9 -5
View File
@@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
get_attn_tp_context,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
@@ -780,8 +779,9 @@ class Glm4MoeLiteModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
@@ -803,8 +803,12 @@ class Glm4MoeLiteModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(hidden_states + residual)
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
+15 -18
View File
@@ -31,8 +31,6 @@ from sglang.srt.layers.communicator import (
MHCLayerCommunicator,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
reduce_output,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -920,7 +918,7 @@ class Glm5NextDecoderLayer(nn.Module):
gemm_output_zero_allocator: BumpAllocator = None,
prev_topk_indices: Optional[torch.Tensor] = None,
):
hidden_states_orig = layer_input_buffer(hidden_states)
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
@@ -1083,14 +1081,7 @@ class Glm5NextModel(nn.Module):
def get_input_embeddings(self) -> torch.Tensor:
return self.embed_tokens
def _prepare_aux_hidden_state(
self, hidden_states: torch.Tensor, residual: Optional[torch.Tensor]
) -> torch.Tensor:
# mHC folds the residual into widened hidden state, so residual remains None
# until hc_contract merges it; only plain residual streams are added here.
aux_hidden_state = (
hidden_states if residual is None else hidden_states + residual
)
def _prepare_aux_hidden_state(self, aux_hidden_state: torch.Tensor) -> torch.Tensor:
if self.dflash_capture and self.config.mhc:
aux_hidden_state = hc_contract(aux_hidden_state, self.config.hc_mult)
return aux_hidden_state
@@ -1112,9 +1103,9 @@ class Glm5NextModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
# mHC carries its residual streams in hidden_states across PP stages.
residual = None if self.config.mhc else pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
@@ -1158,10 +1149,16 @@ class Glm5NextModel(nn.Module):
)
with ctx:
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_state = self._prepare_aux_hidden_state(
hidden_states, residual
)
if self.dflash_capture and self.config.mhc:
# hc_post has completed the streams; contracting them
# creates the retained value without copying the wide input.
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
else:
hidden_states, aux_hidden_state = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
if self.enable_a2a_moe and i > self.first_k_dense_replace:
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
aux_hidden_state, dim=0
+9 -9
View File
@@ -32,7 +32,6 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -704,15 +703,18 @@ class GptOssModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
aux_hidden_states = []
if self.start_layer in self.layers_to_capture:
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
self.layers[self.start_layer].layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -721,12 +723,10 @@ class GptOssModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
if i + 1 in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual
if residual is not None
else hidden_states
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual
)
aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
@@ -787,6 +787,10 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
input_deepstack_embeds[:, start : start + self.hidden_size]
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
+9 -7
View File
@@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -585,16 +584,19 @@ class LagunaModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
@@ -612,7 +614,7 @@ class LagunaModel(nn.Module):
if hidden_states.shape[0] != 0:
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if residual is None:
hidden_states = self.norm(hidden_states)
+7 -2
View File
@@ -749,8 +749,9 @@ class LLaDA2MoeModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -761,6 +762,10 @@ class LLaDA2MoeModel(nn.Module):
forward_batch,
residual,
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
+9 -1
View File
@@ -522,7 +522,11 @@ class Llama4Model(nn.Module):
aux_hidden_states = []
for i in range(len(self.layers)):
if i in self.layers_to_capture:
aux_hidden_states.append(hidden_states + residual)
aux_hidden_states.append(
self.layers[i].layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -530,6 +534,10 @@ class Llama4Model(nn.Module):
forward_batch,
residual,
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
hidden_states, _ = self.norm(hidden_states, residual)
+10 -1
View File
@@ -630,7 +630,11 @@ class LongcatFlashModel(nn.Module):
topk_indices = None
for i in range(total_num_layers):
if i in self.layers_to_capture:
aux_hidden_states.append(hidden_states + residual)
aux_hidden_states.append(
self.layers[i].moe_layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
@@ -642,6 +646,11 @@ class LongcatFlashModel(nn.Module):
topk_indices,
)
last_layer = self.layers[-1]
hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
@@ -275,6 +275,10 @@ class LongcatFlashModelNextN(nn.Module):
positions, hidden_states, forward_batch, residual, zero_allocator
)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
+6 -5
View File
@@ -1040,8 +1040,9 @@ class MiMoV2Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
if forward_batch.can_run_tbo and not self.layers_to_capture:
@@ -1090,7 +1091,7 @@ class MiMoV2Model(nn.Module):
and self.config.num_hidden_layers in self.layers_to_capture
):
aux_hidden_states.append(
hidden_states if residual is None else hidden_states + residual
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
hidden_states_before_norm = None
@@ -1104,8 +1105,8 @@ class MiMoV2Model(nn.Module):
else:
if hidden_states.shape[0] > 0:
if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = (
hidden_states if residual is None else hidden_states + residual
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
hidden_states, residual
)
if residual is None:
hidden_states = self.norm(hidden_states)
+6 -2
View File
@@ -225,11 +225,15 @@ class MiMoV2ModelNextN(nn.Module):
forward_batch=forward_batch,
residual=None,
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = (
hidden_states if residual is None else hidden_states + residual
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
hidden_states, residual
)
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
+3 -2
View File
@@ -1144,8 +1144,9 @@ class MiniMaxM2Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
if forward_batch.can_run_tbo:
+3 -2
View File
@@ -1479,8 +1479,9 @@ class MiniMaxM3Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
if forward_batch.can_run_tbo:
+7 -4
View File
@@ -1114,10 +1114,9 @@ class MossVLTextModel(nn.Module):
for decoder_layer in self.layers:
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
if not skip_cross_attention:
# Fuse residual before cross-attention
if residual is not None:
hidden_states = hidden_states + residual
residual = None
hidden_states, residual = LayerCommunicator.fold(
hidden_states, residual
)
hidden_states = decoder_layer(
hidden_states=hidden_states,
cross_attention_states=cross_attention_states,
@@ -1137,6 +1136,10 @@ class MossVLTextModel(nn.Module):
else:
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
hidden_states, residual = LayerCommunicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states, _ = self.norm(hidden_states, residual)
else:
+13 -30
View File
@@ -31,9 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import UnreducedOutput, reduce_output
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
)
from sglang.srt.layers.layernorm import RMSNorm
@@ -384,10 +382,10 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
def forward(
self,
*,
hidden_states: torch.Tensor | UnreducedOutput,
hidden_states,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
):
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
@@ -466,10 +464,10 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
def forward(
self,
*,
hidden_states: torch.Tensor | UnreducedOutput,
hidden_states,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
):
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
@@ -721,24 +719,6 @@ class NemotronHModel(nn.Module):
self.norm_f = PPMissingLayer(return_tuple=True)
self.layers_to_capture: set[int] = set()
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
if residual is not None and self._owes_attention_sum_at(boundary_idx):
# Reduce a copy so the FFN stage still receives its partial sum.
hidden_states = attn_tp_all_reduce(hidden_states.clone())
# Later norms update the input residual in place.
return hidden_states.clone() if residual is None else hidden_states + residual
def _owes_attention_sum_at(self, boundary_idx: int) -> bool:
"""Whether the value at this boundary is a mixer's partial sum that the
FFN stage after it completes: what the layer after the boundary
declares it takes, or at this rank's end what the last layer declares
it hands on."""
if boundary_idx < self.end_layer:
edge = self.layers[boundary_idx].layer_communicator.stage_edges[0]
else:
edge = self.layers[boundary_idx - 1].layer_communicator.stage_edges[1]
return edge.produced.always_leaves
def forward(
self,
input_ids: torch.Tensor,
@@ -755,16 +735,19 @@ class NemotronHModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
self._capture_hidden_states(hidden_states, residual, i)
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
if not isinstance(layer, Layers):
raise ValueError(f"Unknown layer type: {type(layer)}")
@@ -784,7 +767,7 @@ class NemotronHModel(nn.Module):
)
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
self._capture_hidden_states(hidden_states, residual, self.end_layer)
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if residual is None:
hidden_states = self.norm_f(hidden_states)
+6 -6
View File
@@ -114,9 +114,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states = hidden_states + residual
residual = None
hidden_states, residual = self.layer_communicator.fold(
hidden_states, residual
)
hidden_states = self.final_layernorm(hidden_states)
@@ -198,9 +198,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if residual is not None:
hidden_states = hidden_states + residual
residual = None
hidden_states, residual = self.layer_communicator.fold(
hidden_states, residual
)
hidden_states = self.final_layernorm(hidden_states)
+11 -5
View File
@@ -26,6 +26,11 @@ from sglang.srt.distributed import (
get_pp_indices,
)
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator.residual.access import (
finish_layer_stack,
from_pp,
snapshot,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -408,15 +413,12 @@ class Qwen2Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = from_pp(pp_proxy_tensors)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
)
aux_hidden_states.append(snapshot(hidden_states, residual))
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -424,6 +426,10 @@ class Qwen2Model(nn.Module):
forward_batch,
residual,
)
hidden_states, residual = finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
+8 -2
View File
@@ -1178,8 +1178,9 @@ class Qwen2MoeModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
if forward_batch.can_run_tbo:
@@ -1322,6 +1323,11 @@ class Qwen2MoeForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
+5
View File
@@ -569,6 +569,11 @@ class Qwen3ForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
+7 -6
View File
@@ -46,7 +46,6 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -1821,8 +1820,9 @@ class Qwen3_5ForCausalLM(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
# Pass through decoder layers
@@ -1850,9 +1850,10 @@ class Qwen3_5ForCausalLM(nn.Module):
and layer_idx < 3
):
sep = self.hidden_size * layer_idx
hidden_states = reduce_output(hidden_states)
hidden_states.add_(
input_deepstack_embeds[:, sep : sep + self.hidden_size]
hidden_states, residual = layer.layer_communicator.add_to_output(
hidden_states,
residual,
input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
last_layer = self.layers[self.end_layer - 1]
+5
View File
@@ -1101,6 +1101,11 @@ class Qwen3MoeForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
+10 -3
View File
@@ -1207,8 +1207,9 @@ class Qwen3LLMModel(Qwen3Model):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
for layer_idx, layer in enumerate(
@@ -1217,7 +1218,9 @@ class Qwen3LLMModel(Qwen3Model):
layer_idx = layer_idx + self.start_layer
if layer_idx in self.layers_to_capture:
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
layer.layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
if self.use_hf_deepstack_order:
@@ -1252,6 +1255,10 @@ class Qwen3LLMModel(Qwen3Model):
input_deepstack_embeds[:, sep : sep + self.hidden_size]
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
# Handle deepstack for the last processed layer (HF-order path only).
last_deepstack = (
self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds)
+10 -9
View File
@@ -24,7 +24,6 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import reduce_output
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import get_layer_id
@@ -98,8 +97,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
aux_hidden_states = []
for layer_idx, layer in enumerate(
@@ -107,10 +107,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
):
layer_idx += self.start_layer
if layer_idx in self.layers_to_capture:
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
if self.use_hf_deepstack_order:
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
@@ -140,9 +140,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
hidden_states = reduce_output(hidden_states)
hidden_states.add_(
input_deepstack_embeds[:, sep : sep + self.hidden_size]
hidden_states, residual = layer.layer_communicator.add_to_output(
hidden_states,
residual,
input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
# Handle deepstack for the last processed layer (HF-order path only).
+13 -2
View File
@@ -1154,8 +1154,9 @@ class SarvamMLAModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
@@ -1279,6 +1280,11 @@ class SarvamMLAForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
if forward_batch.residual is None:
hidden_states = self.model.norm(forward_batch.hidden_states)
else:
@@ -1462,6 +1468,11 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
if forward_batch.residual is None:
hidden_states = self.model.norm(forward_batch.hidden_states)
else:
+7 -2
View File
@@ -406,8 +406,9 @@ class SDARModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors.get("residual", None)
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
@@ -415,6 +416,10 @@ class SDARModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
+3 -2
View File
@@ -472,8 +472,9 @@ class SDARMoeModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors.get("residual", None)
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
+4
View File
@@ -478,6 +478,10 @@ class Step3TextModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
+5 -4
View File
@@ -700,8 +700,9 @@ class Step3p5Model(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
@@ -735,8 +736,8 @@ class Step3p5Model(nn.Module):
else:
if hidden_states.shape[0] > 0:
# if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = (
hidden_states if residual is None else hidden_states + residual
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
hidden_states, residual
)
if residual is None:
hidden_states = self.norm(hidden_states)
+2 -2
View File
@@ -121,8 +121,8 @@ class Step3p5AMultiTokenPredictor(nn.Module):
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
# if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = (
hidden_states if residual is None else hidden_states + residual
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
hidden_states, residual
)
if residual is not None:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
+7 -2
View File
@@ -1640,8 +1640,9 @@ class XllmModel(nn.Module):
residual = None
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
for i in range(self.start_layer, self.end_layer):
ctx = (
@@ -1657,6 +1658,10 @@ class XllmModel(nn.Module):
forward_batch,
residual,
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
+2 -1
View File
@@ -12,7 +12,7 @@ from sglang.srt.layers import communicator
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
from sglang.srt.layers.communicator import residual as residual_contract
from sglang.srt.layers.communicator.adapters import attention
from sglang.srt.layers.communicator.residual import add_norm, mhc
from sglang.srt.layers.communicator.residual import access, add_norm, mhc
COMMUNICATOR_MODULES = (
communicator,
@@ -20,6 +20,7 @@ COMMUNICATOR_MODULES = (
output,
attention,
residual_contract,
access,
add_norm,
mhc,
ops,