[Refactor] Capture auxiliary states at residual reads (#41550)

This commit is contained in:
Cheng Wan
2026-09-28 20:16:26 -07:00
committed by GitHub
parent 42f58b4da7
commit cb1fa3c399
29 changed files with 725 additions and 216 deletions
+18 -5
View File
@@ -9,6 +9,18 @@ import torch
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
class AuxHiddenStateList(list):
"""Retain aux snapshots independently of storage reused by later layers."""
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
"""Copy borrowed storage, or adopt a value whose ownership is transferred.
Views and reusable communication buffers are borrowed even when they
are different tensor objects from the source.
"""
self.append(hidden if owned else hidden.clone())
class AuxHiddenStatePacker:
"""Drop-in for the ``[]`` a model collects Eagle3/DFlash captures into.
@@ -17,9 +29,6 @@ class AuxHiddenStatePacker:
Assumes all captures share leading shape and feature size.
"""
# ``append`` copies, so producers need not clone a tensor they later mutate.
copies_on_append = True
def __init__(self, num_captures: int) -> None:
self._num_captures = int(num_captures)
self._buffer: Optional[torch.Tensor] = None
@@ -37,6 +46,10 @@ class AuxHiddenStatePacker:
self._buffer[..., start : start + self._feature_size].copy_(hidden)
self._idx += 1
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
"""Write directly to the final packed buffer, without an intermediate copy."""
self.append(hidden)
def __len__(self) -> int:
return self._idx
@@ -49,8 +62,8 @@ class AuxHiddenStatePacker:
return self._buffer
# What a model hands down the capture path: a plain list, or a packer writing in place.
AuxHiddenStateAccumulator = Union[List[torch.Tensor], AuxHiddenStatePacker]
# Both collectors accept borrowed values through capture; storage belongs to aux.
AuxHiddenStateAccumulator = Union[AuxHiddenStateList, AuxHiddenStatePacker]
def pack_aux_hidden_states(aux_hidden_states: AuxHiddenStates) -> torch.Tensor:
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator.ops import (
_mlp_input_slice,
_mlp_input_without_dp,
_read_input,
move_rows,
tp_reduce_scatter,
)
from sglang.srt.layers.communicator.residual import (
@@ -479,8 +480,6 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
# It may hand back a new residual and leave the one it took unchanged.
may_return_new_residual: bool
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
@@ -548,6 +547,10 @@ class StageEntry(msgspec.Struct, frozen=True):
handoff: Optional[Callable] = None
# The fused kernels prepare tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
capture_preserves_residual: Optional[Callable] = None
class BoundarySteps(msgspec.Struct, frozen=True):
@@ -624,6 +627,19 @@ class Boundary(msgspec.Struct, frozen=True):
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
)
@property
def capture_move_allocates(self) -> bool:
"""Returning through a gather allocates; returning through a cut aliases."""
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
@property
def capture_move(self) -> Optional[Callable]:
if self.edge.residual_to == self.edge.produced.layout:
return None
return partial(
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
)
def make_boundary(
edge: EdgeDecl,
@@ -808,6 +824,8 @@ def _select_boundary_steps(
input_rows=into_attention.input_rows,
input_move=into_attention.input_move,
handoff=attention_handoff,
capture_move=into_attention.capture_move,
capture_move_allocates=into_attention.capture_move_allocates,
),
ffn=StageEntry(
prepare=into_ffn.prepare,
@@ -250,7 +250,6 @@ class CuteDSLFusion:
self._mlp_input_reduce_output_and_update_and_read_residual,
layer,
),
may_return_new_residual=True,
),
)
return ()
+98 -29
View File
@@ -59,9 +59,11 @@ from sglang.srt.layers.communicator.layout import (
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
_all_reduce_then_to_local_tokens,
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
_mlp_input_without_dp,
_redistribute_output,
_reduce_and_redistribute_output_step,
_to_local_tokens,
@@ -79,6 +81,7 @@ from sglang.srt.layers.communicator.residual.access import (
finish_layer_stack,
fold,
from_pp,
norm_output,
snapshot,
written,
)
@@ -94,6 +97,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
@@ -244,6 +248,7 @@ class LayerCommunicator:
add_to_output = staticmethod(add_to_output)
buffer = staticmethod(buffer)
fold = staticmethod(fold)
norm_output = staticmethod(norm_output)
written = staticmethod(written)
finish_layer_stack = staticmethod(finish_layer_stack)
@@ -361,6 +366,46 @@ class LayerCommunicator:
else None
)
# Bind aux ownership to the selected input implementation once.
for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"):
steps = self.__dict__[name]
if steps is None or steps.ffn is None:
continue
prepare = steps.ffn.prepare
if isinstance(prepare, partial):
prepare = prepare.keywords.get("step", prepare)
norm = self.post_attention_layernorm
known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type(
norm
).forward_with_allreduce_fusion in (
RMSNorm.forward_with_allreduce_fusion,
GemmaRMSNorm.forward_with_allreduce_fusion,
)
if (
known_norm
and isinstance(prepare, partial)
and prepare.func is _mlp_input_without_dp
and not prepare.keywords["gathers_residual"]
and steps.ffn.fused
and steps.ffn.fused[0].run
== self._mlp_input_reduce_output_and_update_and_read_residual
):
self.__dict__[name] = msgspec.structs.replace(
steps,
attention=msgspec.structs.replace(
steps.attention,
capture_preserves_residual=self._capture_keeps_residual,
),
)
def _capture_keeps_residual(self, value, forward_batch):
# The CUDA wrapper allocates a fresh residual even on backend fallback.
return (
not _use_aiter
and get_parallel().attn_tp_size > 1
and apply_flashinfer_allreduce_fusion(value.shape[0])
)
def _init_stage(self, stage: "LayerStage") -> None:
"""A layer that is one stage: every batch runs the two boundaries its
declarations give."""
@@ -387,6 +432,7 @@ class LayerCommunicator:
input_move=into.input_move,
handoff=None if is_ffn else _hand_qkv_hook_its_input,
fused=into.fused,
capture_move=into.capture_move,
)
self._steps = BoundarySteps(
attention=None if is_ffn else entry,
@@ -643,7 +689,38 @@ class LayerCommunicator:
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
post_residual_addition: Optional[torch.Tensor] = None,
quant_format: str = "",
capture_output: Optional[Callable] = None,
):
# Aux consumers need a materialized output before the input norm.
# Complete communication here, preserving the existing add+norm kernel
# and its FP32 accumulation. Its residual result also supplies capture.
capture_before_read = capture_output is not None and (
residual is None
or (
isinstance(residual, ResidualStream)
and residual.residual is None
and residual.pending is None
)
or post_residual_addition is not None
)
if capture_output is not None:
hidden_states = (
residual.complete(hidden_states)
if isinstance(residual, ResidualStream)
else reduce_output(hidden_states)
)
if capture_before_read:
# Embeddings precede enter; HF deepstack capture precedes the
# extra addition. Neither is the residual returned by the read.
value, previous = (
residual.finish(hidden_states)
if isinstance(residual, ResidualStream)
else (hidden_states, residual)
)
if previous is None:
capture_output(value)
else:
capture_output(value + previous, owned=True)
hidden_states, residual = self.prepare_attn(
hidden_states,
residual,
@@ -651,6 +728,17 @@ class LayerCommunicator:
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
if capture_output is not None and not capture_before_read:
value = residual.residual
move = self.attn.entry(forward_batch).capture_move
if move is not None:
value = move(value, forward_batch=forward_batch)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
capture_output(
value,
owned=self.attn.entry(forward_batch).capture_move_allocates
or (move is None and keeps is not None and keeps(value, forward_batch)),
)
if captured_last_layer_outputs is not None:
residual_value = residual.residual
move = self.attn.entry(forward_batch).input_move
@@ -663,36 +751,18 @@ class LayerCommunicator:
context=self._context,
)
)
if (
gathered_last_layer_output is residual_value
# An accumulator that copies on append already holds a snapshot.
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
and not self._post_attn_residual_is_read_only(
residual_value, forward_batch
)
):
gathered_last_layer_output = residual_value.clone()
captured_last_layer_outputs.append(gathered_last_layer_output)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
captured_last_layer_outputs.capture(
gathered_last_layer_output,
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
or (
move is None
and keeps is not None
and keeps(residual_value, forward_batch)
),
)
return hidden_states, residual
def _post_attn_residual_is_read_only(
self, residual: torch.Tensor, forward_batch: ForwardBatch
) -> bool:
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
Of the base fused entry's kernels only flashinfer's writes a fresh
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
kernel and every plain norm fold into ``residual`` in place. It runs when
``prepare_mlp`` selected a fused entry that may return a new residual and
the batch is not input-scattered.
"""
return (
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
def publish_attn_lora_layout(self) -> None:
"""Attention consumes the DP-local token batch."""
if self._publish_lora_layout:
@@ -829,7 +899,6 @@ class LayerCommunicator:
FusedMlpInput(
completes=SumGroup.ATTN_TP,
run=self._mlp_input_reduce_output_and_update_and_read_residual,
may_return_new_residual=True,
),
)
@@ -100,6 +100,23 @@ def finish_layer_stack(
return reduce_output(hidden_states), residual
def norm_output(hidden_states, residual, norm, capture_output=None):
"""The final norm and an optional capture of the same updated residual.
Keep add and norm together: normalizing a separately rounded residual is
not numerically equivalent to the fused kernel's FP32 accumulation. The
capture callback owns retention of the borrowed residual storage.
"""
if residual is None:
if capture_output is not None:
capture_output(hidden_states)
return norm(hidden_states)
hidden_states, residual = norm(hidden_states, residual)
if capture_output is not None:
capture_output(residual)
return hidden_states
def from_pp(
tensors: PPProxyTensors,
*,
@@ -235,8 +235,8 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
tensor, output_buffer.tensors[key], num_tokens
)
return
if isinstance(output, (list, tuple)) and isinstance(
output_buffer, type(output)
if (isinstance(output, list) and isinstance(output_buffer, list)) or (
isinstance(output, tuple) and isinstance(output_buffer, tuple)
):
if len(output) != len(output_buffer):
raise ValueError(
+11 -9
View File
@@ -31,6 +31,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -722,7 +726,8 @@ class BailingMoEBlock(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
@@ -730,6 +735,7 @@ class BailingMoEBlock(nn.Module):
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
)
)
@@ -822,16 +828,9 @@ class BailingMoEModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -843,6 +842,9 @@ class BailingMoEModel(nn.Module):
if getattr(layer, "_is_layer_to_capture", False)
else None
),
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
+29 -13
View File
@@ -28,6 +28,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1178,10 +1179,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if not forward_batch.forward_mode.is_idle():
@@ -1370,7 +1377,11 @@ class BailingMoELinearModel(nn.Module):
and capture_mode.need_capture()
)
if capture_aux:
dspark_aux_hidden_states: List[torch.Tensor] = []
dspark_aux_hidden_states = AuxHiddenStateList()
def capture_output(value, *, owned=False):
if value.shape[0] != 0:
dspark_aux_hidden_states.capture(value, owned=owned)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -1381,13 +1392,12 @@ class BailingMoELinearModel(nn.Module):
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
capture_output=capture_output
if capture_aux
and i > self.start_layer
and i - 1 in self.layers_to_capture
else None,
)
if capture_aux and i in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, skip_empty=True
)
if snapshot is not None:
dspark_aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
@@ -1399,10 +1409,16 @@ class BailingMoELinearModel(nn.Module):
)
else:
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
capture_output
if capture_aux and self.end_layer - 1 in self.layers_to_capture
else None
),
)
if capture_aux and len(dspark_aux_hidden_states) > 0:
return hidden_states, dspark_aux_hidden_states
return hidden_states
+14 -13
View File
@@ -38,6 +38,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -936,13 +937,17 @@ class Glm4MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -1103,22 +1108,18 @@ class Glm4MoeModel(nn.Module):
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+14 -13
View File
@@ -32,6 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -625,12 +626,16 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
getattr(self, "_gfx95_quant_format", ""),
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=getattr(self, "_gfx95_quant_format", ""),
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -800,16 +805,9 @@ class Glm4MoeLiteModel(nn.Module):
normal_end_layer = self.first_k_dense_replace
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -817,6 +815,9 @@ class Glm4MoeLiteModel(nn.Module):
forward_batch,
residual,
zero_allocator,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+22 -20
View File
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_distribution import (
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -918,13 +919,17 @@ class Glm5NextDecoderLayer(nn.Module):
zero_allocator: Optional[BumpAllocator] = None,
gemm_output_zero_allocator: BumpAllocator = None,
prev_topk_indices: Optional[torch.Tensor] = None,
capture_output=None,
):
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -1139,7 +1144,7 @@ class Glm5NextModel(nn.Module):
normal_end_layer = self.first_k_dense_replace
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
topk_indices = None
for i in range(normal_start_layer, normal_end_layer):
# NOTE: torch dynamo does not support graph break in context manager
@@ -1149,22 +1154,16 @@ class Glm5NextModel(nn.Module):
else get_global_expert_distribution_recorder().with_current_layer(i)
)
with ctx:
if i in self.layers_to_capture:
if self.dflash_capture and self.config.mhc:
# hc_post has completed the streams; contracting them
# creates the retained value without copying the wide input.
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
else:
hidden_states, aux_hidden_state = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
def capture_output(aux_hidden_state, *, owned=False):
aux_hidden_state = self._prepare_aux_hidden_state(aux_hidden_state)
owned = owned or (self.dflash_capture and self.config.mhc)
if self.enable_a2a_moe and i > self.first_k_dense_replace:
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
aux_hidden_state, dim=0
)
aux_hidden_states.append(aux_hidden_state)
group = get_parallel().attn_tp_group
aux_hidden_state = group.all_gather(aux_hidden_state, dim=0)
owned = owned or group.world_size > 1
aux_hidden_states.capture(aux_hidden_state, owned=owned)
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
positions,
@@ -1174,6 +1173,9 @@ class Glm5NextModel(nn.Module):
zero_allocator,
gemm_output_zero_allocator,
prev_topk_indices=topk_indices,
capture_output=capture_output
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+31 -19
View File
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -618,9 +619,15 @@ class GptOssDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if hidden_states.shape[0] != 0:
@@ -710,24 +717,19 @@ class GptOssModel(nn.Module):
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
aux_hidden_states = []
if self.start_layer in self.layers_to_capture:
aux_hidden_states.append(
self.layers[self.start_layer].layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if i + 1 in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual
)
aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
@@ -741,10 +743,20 @@ class GptOssModel(nn.Module):
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
)
elif self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if len(aux_hidden_states) == 0:
return hidden_states
+6 -2
View File
@@ -10,6 +10,10 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
)
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -644,7 +648,7 @@ class InternS2MobiusAttentionDecoderLayer(
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
meta_mlp: nn.ModuleList,
captured_last_layer_outputs: list[torch.Tensor] | None = None,
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
**kwargs,
):
del kwargs
@@ -764,7 +768,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(self.layers):
hidden_states, residual = layer(
positions=positions,
+27 -19
View File
@@ -19,6 +19,7 @@ from torch import nn
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -475,9 +476,15 @@ class LagunaDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -589,18 +596,17 @@ class LagunaModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
@@ -613,14 +619,16 @@ class LagunaModel(nn.Module):
)
if hidden_states.shape[0] != 0:
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
)
if len(aux_hidden_states) == 0:
return hidden_states
return hidden_states, aux_hidden_states
+7 -3
View File
@@ -18,7 +18,7 @@
import logging
from contextlib import nullcontext
from functools import lru_cache
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
import torch
import triton
@@ -30,6 +30,10 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -997,7 +1001,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states, residual = (
@@ -1149,7 +1153,7 @@ class MiniMaxM2Model(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+6 -2
View File
@@ -35,6 +35,10 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1369,7 +1373,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> torch.Tensor:
hidden_states, residual = (
@@ -1484,7 +1488,7 @@ class MiniMaxM3Model(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+6 -2
View File
@@ -40,6 +40,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1070,7 +1074,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -1183,7 +1187,7 @@ class Qwen2MoeModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+8 -3
View File
@@ -23,7 +23,6 @@ import torch
import torch.nn as nn
import triton
# Layers - Attention
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
fused_qkvzba_split_reshape_cat_contiguous,
@@ -41,6 +40,12 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
# Layers - Attention
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1540,7 +1545,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
hidden_states, residual = (
@@ -1825,7 +1830,7 @@ class Qwen3_5ForCausalLM(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
# Pass through decoder layers
for layer_idx in range(self.start_layer, self.end_layer):
layer = self.layers[layer_idx]
+4 -1
View File
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -882,7 +883,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -892,6 +894,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
)
+7 -3
View File
@@ -15,6 +15,10 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -550,7 +554,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
forward_batch = kwargs.get("forward_batch", None)
@@ -826,7 +830,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs: Any,
):
hidden_states, residual = (
@@ -931,7 +935,7 @@ class Qwen3NextModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(len(self.layers)):
layer = self.layers[i]
with get_global_expert_distribution_recorder().with_current_layer(i):
+8 -6
View File
@@ -24,6 +24,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -102,16 +103,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer]
):
layer_idx += self.start_layer
if layer_idx in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
if self.use_hf_deepstack_order:
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
@@ -126,6 +122,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
forward_batch,
residual,
post_residual_addition=deepstack_embeds,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
)
else:
# Inference path: add deepstack directly to hidden_states at the end of the layer
@@ -135,6 +134,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
)
if (
input_deepstack_embeds is not None
+2 -1
View File
@@ -21,6 +21,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather,
@@ -1748,7 +1749,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
if self.has_ple
else None
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
if i + 1 < self.end_layer: