[Refactor] Centralize decoder output access (#41548)
This commit is contained in:
@@ -59,7 +59,6 @@ from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
layer_input_buffer,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
@@ -119,7 +118,6 @@ __all__ = [
|
||||
"enable_moe_dense_fully_dp",
|
||||
"get_attn_tp_context",
|
||||
"input_scattered_layer_sides",
|
||||
"layer_input_buffer",
|
||||
"make_boundary",
|
||||
"make_output_boundary",
|
||||
"moe_cp_gathers_sparse_moe_input",
|
||||
|
||||
@@ -100,6 +100,14 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
update: StageUpdate = ADD
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""What a decoder layer declares about its attention, its FFN and the rows
|
||||
between them; ``decoder_layer_edges`` turns it into the layer's
|
||||
@@ -107,10 +115,8 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageInput
|
||||
attention_output: StageOutput
|
||||
ffn: StageInput
|
||||
ffn_output: StageOutput
|
||||
attention: StageDecl
|
||||
ffn: StageDecl
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
@@ -122,14 +128,6 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
@@ -167,7 +165,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
sides.input_rows,
|
||||
group=sides.input_owes,
|
||||
always_leaves=owes,
|
||||
update=sides.ffn_output.update,
|
||||
update=sides.ffn.output.update,
|
||||
)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
@@ -179,19 +177,19 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
return DecoderLayerEdges(
|
||||
into_attention=EdgeDecl(
|
||||
produced=arrived,
|
||||
need=sides.attention,
|
||||
need=sides.attention.input,
|
||||
residual=sides.input_rows,
|
||||
residual_to=attention_rows,
|
||||
),
|
||||
into_ffn=EdgeDecl(
|
||||
produced=sides.attention_output,
|
||||
need=sides.ffn,
|
||||
produced=sides.attention.output,
|
||||
need=sides.ffn.input,
|
||||
residual=attention_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
),
|
||||
out_of_ffn=EdgeDecl(
|
||||
produced=sides.ffn_output,
|
||||
produced=sides.ffn.output,
|
||||
need=StageInput(sides.output_rows),
|
||||
residual=sides.ffn_residual_rows,
|
||||
residual_to=sides.output_rows,
|
||||
@@ -207,12 +205,14 @@ def with_residual(
|
||||
replace = msgspec.structs.replace
|
||||
return replace(
|
||||
sides,
|
||||
attention=replace(sides.attention, read=residual.attention_read),
|
||||
attention_output=replace(
|
||||
sides.attention_output, update=residual.attention_update
|
||||
attention=StageDecl(
|
||||
input=replace(sides.attention.input, read=residual.attention_read),
|
||||
output=replace(sides.attention.output, update=residual.attention_update),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=replace(sides.ffn.input, read=residual.ffn_read),
|
||||
output=replace(sides.ffn.output, update=residual.ffn_update),
|
||||
),
|
||||
ffn=replace(sides.ffn, read=residual.ffn_read),
|
||||
ffn_output=replace(sides.ffn_output, update=residual.ffn_update),
|
||||
)
|
||||
|
||||
|
||||
@@ -270,14 +270,17 @@ def sequence_parallel_layer_sides(
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
# qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out.
|
||||
attention=StageDecl(
|
||||
input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
|
||||
output=StageOutput(local),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
output=StageOutput(local),
|
||||
),
|
||||
ffn_output=StageOutput(local),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
@@ -311,19 +314,21 @@ def scattered_residual_layer_sides(
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
ffn_output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
ffn=StageDecl(
|
||||
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
@@ -347,18 +352,20 @@ def input_scattered_layer_sides(
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next layer's input completes the sum; the last layer's FFN does.
|
||||
ffn_output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
ffn=StageDecl(
|
||||
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next input completes the sum; the last FFN completes its own.
|
||||
output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
@@ -412,24 +419,26 @@ def decoder_layer_sides(
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
output=StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if attention_tp else None,
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if attention_tp else None,
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
ffn=StageInput(ffn),
|
||||
ffn_output=(
|
||||
StageOutput(ffn)
|
||||
ffn=StageDecl(
|
||||
input=StageInput(ffn),
|
||||
output=StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
@@ -437,7 +446,7 @@ def decoder_layer_sides(
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
)
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=(
|
||||
|
||||
@@ -68,11 +68,18 @@ from sglang.srt.layers.communicator.ops import (
|
||||
move_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.communicator.residual.access import (
|
||||
add_to_output,
|
||||
buffer,
|
||||
finish_layer_stack,
|
||||
fold,
|
||||
from_pp,
|
||||
snapshot,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
PLAIN_RESIDUAL,
|
||||
aiter_all_reduce_fusion_enabled_for,
|
||||
@@ -221,6 +228,11 @@ class LayerFusions(Protocol):
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
add_to_output = staticmethod(add_to_output)
|
||||
buffer = staticmethod(buffer)
|
||||
fold = staticmethod(fold)
|
||||
finish_layer_stack = staticmethod(finish_layer_stack)
|
||||
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
_publish_lora_layout: bool = False
|
||||
@@ -229,6 +241,7 @@ class LayerCommunicator:
|
||||
_residual: LayerResidual = PLAIN_RESIDUAL
|
||||
# The fused kernels a backend gives the layer, if any.
|
||||
fusions: Optional[LayerFusions] = None
|
||||
stage_edges = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -534,14 +547,14 @@ class LayerCommunicator:
|
||||
if self._residual.attention_update.adds_plainly:
|
||||
scattered = input_scattered_layer_sides(
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
ffn_group=sides.ffn.output.group,
|
||||
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
|
||||
)
|
||||
handoff = _hand_qkv_hook_its_input
|
||||
else:
|
||||
scattered = scattered_residual_layer_sides(
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
ffn_group=sides.ffn.output.group,
|
||||
is_first_layer=self.layer_facts.is_first_layer,
|
||||
is_last_layer=self.is_last_layer,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
@@ -566,6 +579,39 @@ class LayerCommunicator:
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def snapshot(self, hidden_states, residual, *, at_input: bool = False):
|
||||
group = None
|
||||
if residual is not None and self.stage_edges is not None:
|
||||
edge = self.stage_edges[0 if at_input else 1]
|
||||
if edge.produced.always_leaves:
|
||||
group = _sum_group(edge.produced.group)
|
||||
return snapshot(hidden_states, residual, group=group)
|
||||
|
||||
def capture_output(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
*,
|
||||
at_input: bool = False,
|
||||
skip_empty: bool = False,
|
||||
):
|
||||
"""Complete the carried output on the main path, then snapshot it.
|
||||
Keep this explicit completion distinct from a read-only snapshot: the
|
||||
next prepare must not select a fusion for a sum already completed here."""
|
||||
if skip_empty:
|
||||
storage = buffer(hidden_states)
|
||||
if storage is not None and storage.shape[0] == 0:
|
||||
return hidden_states, None
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input)
|
||||
|
||||
def from_pp(self, tensors, *, allow_missing_residual: bool = False):
|
||||
return from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=self._residual.ffn_update.at_producer,
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
|
||||
def prepare_attn_and_capture_last_layer_outputs(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -977,7 +1023,7 @@ class LayerCommunicator:
|
||||
if self._declared is None:
|
||||
raise NotImplementedError("a branch on a layer that is one stage")
|
||||
sides = self._declared
|
||||
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
|
||||
return sides.ffn.input.layout, sides.ffn_residual_rows, sides.output_rows
|
||||
|
||||
def branch_input(
|
||||
self,
|
||||
@@ -1022,23 +1068,6 @@ class LayerCommunicator:
|
||||
residual = move_rows(residual, rows, to, forward_batch)
|
||||
return contribution + hidden_states, residual
|
||||
|
||||
def finish_layer_stack(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
final_norm_takes_handoff: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers. A final norm
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
def _select_ffn_exit_fusions(
|
||||
self,
|
||||
) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]:
|
||||
@@ -1304,7 +1333,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
# MHC has not been run with an FFN input gathered over attention CP.
|
||||
if (
|
||||
TokenAxis.ATTN_CP
|
||||
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
|
||||
in sides.attention.output.layout.sharded - sides.ffn.input.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
|
||||
|
||||
@@ -102,15 +102,6 @@ def tp_reduce_scatter(
|
||||
return output, residual
|
||||
|
||||
|
||||
def layer_input_buffer(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
) -> torch.Tensor:
|
||||
"""The tensor holding a layer's input, for reusing its memory without reading it."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return hidden_states.partial
|
||||
return hidden_states
|
||||
|
||||
|
||||
class CommunicateSimpleFn:
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Access to a layer output outside the next stage's prepare call."""
|
||||
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
|
||||
|
||||
def buffer(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Storage that can be reused after prepare consumes the input. A finalize
|
||||
handoff has no reusable layer-output tensor. This does not complete or read
|
||||
the value held in the storage."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return hidden_states.partial
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
return None
|
||||
return hidden_states
|
||||
|
||||
|
||||
def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple:
|
||||
"""Complete the output before adding an extra contribution exactly once.
|
||||
Leave its residual update for the next prepare call."""
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states.add_(extra)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]:
|
||||
"""Finish a plain layer output and fold its residual into a complete value.
|
||||
The following stage receives it with no outstanding residual addition."""
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
return hidden_states, None
|
||||
|
||||
|
||||
def finish_layer_stack(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
final_norm_takes_handoff: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers. A final norm
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
|
||||
def from_pp(
|
||||
tensors: PPProxyTensors,
|
||||
*,
|
||||
residual_in_hidden: bool = False,
|
||||
allow_missing_residual: bool = False,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Receive the layer-stack state without reducing a declared partial sum.
|
||||
MHC carries its written streams in hidden_states; ordinary layers receive
|
||||
a separate residual tensor."""
|
||||
if residual_in_hidden:
|
||||
residual = None
|
||||
elif allow_missing_residual:
|
||||
residual = tensors.tensors.get("residual")
|
||||
else:
|
||||
residual = tensors["residual"]
|
||||
return tensors["hidden_states"], residual
|
||||
|
||||
|
||||
def snapshot(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
*,
|
||||
group: Optional[GroupCoordinator] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Copy a complete output with its plain residual. A statically declared
|
||||
sum is reduced on a copy; the main output and residual remain unchanged."""
|
||||
if group is not None:
|
||||
hidden_states = group.all_reduce(hidden_states.clone())
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
@@ -35,7 +35,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -818,17 +817,20 @@ class BailingMoEModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
|
||||
@@ -983,8 +983,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
|
||||
|
||||
@@ -32,8 +32,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1344,8 +1342,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = hidden_states.device
|
||||
@@ -1382,16 +1381,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
if (
|
||||
capture_aux
|
||||
and i in self.layers_to_capture
|
||||
and layer_input_buffer(hidden_states).shape[0] != 0
|
||||
):
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
if residual is None:
|
||||
dspark_aux_hidden_states.append(hidden_states)
|
||||
else:
|
||||
dspark_aux_hidden_states.append(hidden_states + residual)
|
||||
if capture_aux and i in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, skip_empty=True
|
||||
)
|
||||
if snapshot is not None:
|
||||
dspark_aux_hidden_states.append(snapshot)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
|
||||
@@ -72,7 +72,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
@@ -2706,7 +2705,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
next_full_attention_layer_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_orig = layer_input_buffer(hidden_states)
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
@@ -3032,8 +3031,9 @@ class DeepseekV2Model(nn.Module):
|
||||
initial_topk_indices = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
|
||||
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
|
||||
if not self.pp_group.is_first_rank:
|
||||
|
||||
@@ -1768,8 +1768,9 @@ class Dots3Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
|
||||
@@ -432,6 +432,10 @@ class FalconH1Model(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
@@ -561,8 +561,9 @@ class GigaChat35Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
zero_allocator = BumpAllocator(
|
||||
@@ -580,6 +581,10 @@ class GigaChat35Model(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
|
||||
@@ -115,10 +115,14 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states_before_norm = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states_before_norm = self.decoder.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
|
||||
@@ -42,7 +42,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
@@ -1088,8 +1087,9 @@ class Glm4MoeModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
@@ -1106,8 +1106,12 @@ class Glm4MoeModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
|
||||
@@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
@@ -780,8 +779,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
@@ -803,8 +803,12 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
|
||||
@@ -31,8 +31,6 @@ from sglang.srt.layers.communicator import (
|
||||
MHCLayerCommunicator,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -920,7 +918,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
):
|
||||
hidden_states_orig = layer_input_buffer(hidden_states)
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
@@ -1083,14 +1081,7 @@ class Glm5NextModel(nn.Module):
|
||||
def get_input_embeddings(self) -> torch.Tensor:
|
||||
return self.embed_tokens
|
||||
|
||||
def _prepare_aux_hidden_state(
|
||||
self, hidden_states: torch.Tensor, residual: Optional[torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
# mHC folds the residual into widened hidden state, so residual remains None
|
||||
# until hc_contract merges it; only plain residual streams are added here.
|
||||
aux_hidden_state = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
def _prepare_aux_hidden_state(self, aux_hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
if self.dflash_capture and self.config.mhc:
|
||||
aux_hidden_state = hc_contract(aux_hidden_state, self.config.hc_mult)
|
||||
return aux_hidden_state
|
||||
@@ -1112,9 +1103,9 @@ class Glm5NextModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
# mHC carries its residual streams in hidden_states across PP stages.
|
||||
residual = None if self.config.mhc else pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
@@ -1158,10 +1149,16 @@ class Glm5NextModel(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(
|
||||
hidden_states, residual
|
||||
)
|
||||
if self.dflash_capture and self.config.mhc:
|
||||
# hc_post has completed the streams; contracting them
|
||||
# creates the retained value without copying the wide input.
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
|
||||
else:
|
||||
hidden_states, aux_hidden_state = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
if self.enable_a2a_moe and i > self.first_k_dense_replace:
|
||||
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
|
||||
aux_hidden_state, dim=0
|
||||
|
||||
@@ -32,7 +32,6 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -704,15 +703,18 @@ class GptOssModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
# Capture hidden-state boundaries: boundary 0 is the embedding output,
|
||||
# and boundary i + 1 is the output after transformer block i.
|
||||
aux_hidden_states = []
|
||||
if self.start_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
self.layers[self.start_layer].layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -721,12 +723,10 @@ class GptOssModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
if i + 1 in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual
|
||||
if residual is not None
|
||||
else hidden_states
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
|
||||
@@ -787,6 +787,10 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
input_deepstack_embeds[:, start : start + self.hidden_size]
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -585,16 +584,19 @@ class LagunaModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
@@ -612,7 +614,7 @@ class LagunaModel(nn.Module):
|
||||
if hidden_states.shape[0] != 0:
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -749,8 +749,9 @@ class LLaDA2MoeModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -761,6 +762,10 @@ class LLaDA2MoeModel(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
|
||||
@@ -522,7 +522,11 @@ class Llama4Model(nn.Module):
|
||||
aux_hidden_states = []
|
||||
for i in range(len(self.layers)):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -530,6 +534,10 @@ class Llama4Model(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
|
||||
@@ -630,7 +630,11 @@ class LongcatFlashModel(nn.Module):
|
||||
topk_indices = None
|
||||
for i in range(total_num_layers):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].moe_layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
@@ -642,6 +646,11 @@ class LongcatFlashModel(nn.Module):
|
||||
topk_indices,
|
||||
)
|
||||
|
||||
last_layer = self.layers[-1]
|
||||
hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -275,6 +275,10 @@ class LongcatFlashModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
|
||||
@@ -1040,8 +1040,9 @@ class MiMoV2Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
|
||||
if forward_batch.can_run_tbo and not self.layers_to_capture:
|
||||
@@ -1090,7 +1091,7 @@ class MiMoV2Model(nn.Module):
|
||||
and self.config.num_hidden_layers in self.layers_to_capture
|
||||
):
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
|
||||
hidden_states_before_norm = None
|
||||
@@ -1104,8 +1105,8 @@ class MiMoV2Model(nn.Module):
|
||||
else:
|
||||
if hidden_states.shape[0] > 0:
|
||||
if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -225,11 +225,15 @@ class MiMoV2ModelNextN(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
|
||||
@@ -1144,8 +1144,9 @@ class MiniMaxM2Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
|
||||
@@ -1479,8 +1479,9 @@ class MiniMaxM3Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
|
||||
@@ -1114,10 +1114,9 @@ class MossVLTextModel(nn.Module):
|
||||
for decoder_layer in self.layers:
|
||||
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
|
||||
if not skip_cross_attention:
|
||||
# Fuse residual before cross-attention
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = None
|
||||
hidden_states, residual = LayerCommunicator.fold(
|
||||
hidden_states, residual
|
||||
)
|
||||
hidden_states = decoder_layer(
|
||||
hidden_states=hidden_states,
|
||||
cross_attention_states=cross_attention_states,
|
||||
@@ -1137,6 +1136,10 @@ class MossVLTextModel(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
|
||||
|
||||
hidden_states, residual = LayerCommunicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
else:
|
||||
|
||||
@@ -31,9 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import UnreducedOutput, reduce_output
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -384,10 +382,10 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor | UnreducedOutput,
|
||||
hidden_states,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
|
||||
):
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
@@ -466,10 +464,10 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor | UnreducedOutput,
|
||||
hidden_states,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
|
||||
):
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
@@ -721,24 +719,6 @@ class NemotronHModel(nn.Module):
|
||||
self.norm_f = PPMissingLayer(return_tuple=True)
|
||||
self.layers_to_capture: set[int] = set()
|
||||
|
||||
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
|
||||
if residual is not None and self._owes_attention_sum_at(boundary_idx):
|
||||
# Reduce a copy so the FFN stage still receives its partial sum.
|
||||
hidden_states = attn_tp_all_reduce(hidden_states.clone())
|
||||
# Later norms update the input residual in place.
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
|
||||
def _owes_attention_sum_at(self, boundary_idx: int) -> bool:
|
||||
"""Whether the value at this boundary is a mixer's partial sum that the
|
||||
FFN stage after it completes: what the layer after the boundary
|
||||
declares it takes, or at this rank's end what the last layer declares
|
||||
it hands on."""
|
||||
if boundary_idx < self.end_layer:
|
||||
edge = self.layers[boundary_idx].layer_communicator.stage_edges[0]
|
||||
else:
|
||||
edge = self.layers[boundary_idx - 1].layer_communicator.stage_edges[1]
|
||||
return edge.produced.always_leaves
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -755,16 +735,19 @@ class NemotronHModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
self._capture_hidden_states(hidden_states, residual, i)
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
if not isinstance(layer, Layers):
|
||||
raise ValueError(f"Unknown layer type: {type(layer)}")
|
||||
@@ -784,7 +767,7 @@ class NemotronHModel(nn.Module):
|
||||
)
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self._capture_hidden_states(hidden_states, residual, self.end_layer)
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm_f(hidden_states)
|
||||
|
||||
@@ -114,9 +114,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = None
|
||||
hidden_states, residual = self.layer_communicator.fold(
|
||||
hidden_states, residual
|
||||
)
|
||||
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
@@ -198,9 +198,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = None
|
||||
hidden_states, residual = self.layer_communicator.fold(
|
||||
hidden_states, residual
|
||||
)
|
||||
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
)
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator.residual.access import (
|
||||
finish_layer_stack,
|
||||
from_pp,
|
||||
snapshot,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -408,15 +413,12 @@ class Qwen2Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
)
|
||||
aux_hidden_states.append(snapshot(hidden_states, residual))
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -424,6 +426,10 @@ class Qwen2Model(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
|
||||
hidden_states, residual = finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
|
||||
@@ -1178,8 +1178,9 @@ class Qwen2MoeModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
@@ -1322,6 +1323,11 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
|
||||
@@ -569,6 +569,11 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
|
||||
@@ -46,7 +46,6 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -1821,8 +1820,9 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
# Pass through decoder layers
|
||||
@@ -1850,9 +1850,10 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
and layer_idx < 3
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
hidden_states, residual = layer.layer_communicator.add_to_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size],
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
|
||||
@@ -1101,6 +1101,11 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
|
||||
@@ -1207,8 +1207,9 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for layer_idx, layer in enumerate(
|
||||
@@ -1217,7 +1218,9 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
layer_idx = layer_idx + self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
layer.layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
|
||||
if self.use_hf_deepstack_order:
|
||||
@@ -1252,6 +1255,10 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
|
||||
# Handle deepstack for the last processed layer (HF-order path only).
|
||||
last_deepstack = (
|
||||
self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds)
|
||||
|
||||
@@ -24,7 +24,6 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import reduce_output
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
@@ -98,8 +97,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
aux_hidden_states = []
|
||||
for layer_idx, layer in enumerate(
|
||||
@@ -107,10 +107,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
):
|
||||
layer_idx += self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
|
||||
if self.use_hf_deepstack_order:
|
||||
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
|
||||
@@ -140,9 +140,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
and layer_idx in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
hidden_states, residual = layer.layer_communicator.add_to_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size],
|
||||
)
|
||||
|
||||
# Handle deepstack for the last processed layer (HF-order path only).
|
||||
|
||||
@@ -1154,8 +1154,9 @@ class SarvamMLAModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
@@ -1279,6 +1280,11 @@ class SarvamMLAForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
if forward_batch.residual is None:
|
||||
hidden_states = self.model.norm(forward_batch.hidden_states)
|
||||
else:
|
||||
@@ -1462,6 +1468,11 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
if forward_batch.residual is None:
|
||||
hidden_states = self.model.norm(forward_batch.hidden_states)
|
||||
else:
|
||||
|
||||
@@ -406,8 +406,9 @@ class SDARModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors.get("residual", None)
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
@@ -415,6 +416,10 @@ class SDARModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
|
||||
@@ -472,8 +472,9 @@ class SDARMoeModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors.get("residual", None)
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -478,6 +478,10 @@ class Step3TextModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -700,8 +700,9 @@ class Step3p5Model(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
@@ -735,8 +736,8 @@ class Step3p5Model(nn.Module):
|
||||
else:
|
||||
if hidden_states.shape[0] > 0:
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states_before_norm = last_layer.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -121,8 +121,8 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
hidden_states_before_norm = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot(
|
||||
hidden_states, residual
|
||||
)
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
|
||||
@@ -1640,8 +1640,9 @@ class XllmModel(nn.Module):
|
||||
residual = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
ctx = (
|
||||
@@ -1657,6 +1658,10 @@ class XllmModel(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.srt.layers import communicator
|
||||
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
|
||||
from sglang.srt.layers.communicator import residual as residual_contract
|
||||
from sglang.srt.layers.communicator.adapters import attention
|
||||
from sglang.srt.layers.communicator.residual import add_norm, mhc
|
||||
from sglang.srt.layers.communicator.residual import access, add_norm, mhc
|
||||
|
||||
COMMUNICATOR_MODULES = (
|
||||
communicator,
|
||||
@@ -20,6 +20,7 @@ COMMUNICATOR_MODULES = (
|
||||
output,
|
||||
attention,
|
||||
residual_contract,
|
||||
access,
|
||||
add_norm,
|
||||
mhc,
|
||||
ops,
|
||||
|
||||
Reference in New Issue
Block a user