[Refactor] Capture auxiliary states at residual reads (#41550)

This commit is contained in:
Cheng Wan
2026-09-28 20:16:26 -07:00
committed by GitHub
parent 42f58b4da7
commit cb1fa3c399
29 changed files with 725 additions and 216 deletions
+18 -5
View File
@@ -9,6 +9,18 @@ import torch
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
class AuxHiddenStateList(list):
"""Retain aux snapshots independently of storage reused by later layers."""
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
"""Copy borrowed storage, or adopt a value whose ownership is transferred.
Views and reusable communication buffers are borrowed even when they
are different tensor objects from the source.
"""
self.append(hidden if owned else hidden.clone())
class AuxHiddenStatePacker:
"""Drop-in for the ``[]`` a model collects Eagle3/DFlash captures into.
@@ -17,9 +29,6 @@ class AuxHiddenStatePacker:
Assumes all captures share leading shape and feature size.
"""
# ``append`` copies, so producers need not clone a tensor they later mutate.
copies_on_append = True
def __init__(self, num_captures: int) -> None:
self._num_captures = int(num_captures)
self._buffer: Optional[torch.Tensor] = None
@@ -37,6 +46,10 @@ class AuxHiddenStatePacker:
self._buffer[..., start : start + self._feature_size].copy_(hidden)
self._idx += 1
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
"""Write directly to the final packed buffer, without an intermediate copy."""
self.append(hidden)
def __len__(self) -> int:
return self._idx
@@ -49,8 +62,8 @@ class AuxHiddenStatePacker:
return self._buffer
# What a model hands down the capture path: a plain list, or a packer writing in place.
AuxHiddenStateAccumulator = Union[List[torch.Tensor], AuxHiddenStatePacker]
# Both collectors accept borrowed values through capture; storage belongs to aux.
AuxHiddenStateAccumulator = Union[AuxHiddenStateList, AuxHiddenStatePacker]
def pack_aux_hidden_states(aux_hidden_states: AuxHiddenStates) -> torch.Tensor:
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator.ops import (
_mlp_input_slice,
_mlp_input_without_dp,
_read_input,
move_rows,
tp_reduce_scatter,
)
from sglang.srt.layers.communicator.residual import (
@@ -479,8 +480,6 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
# It may hand back a new residual and leave the one it took unchanged.
may_return_new_residual: bool
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
@@ -548,6 +547,10 @@ class StageEntry(msgspec.Struct, frozen=True):
handoff: Optional[Callable] = None
# The fused kernels prepare tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
capture_preserves_residual: Optional[Callable] = None
class BoundarySteps(msgspec.Struct, frozen=True):
@@ -624,6 +627,19 @@ class Boundary(msgspec.Struct, frozen=True):
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
)
@property
def capture_move_allocates(self) -> bool:
"""Returning through a gather allocates; returning through a cut aliases."""
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
@property
def capture_move(self) -> Optional[Callable]:
if self.edge.residual_to == self.edge.produced.layout:
return None
return partial(
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
)
def make_boundary(
edge: EdgeDecl,
@@ -808,6 +824,8 @@ def _select_boundary_steps(
input_rows=into_attention.input_rows,
input_move=into_attention.input_move,
handoff=attention_handoff,
capture_move=into_attention.capture_move,
capture_move_allocates=into_attention.capture_move_allocates,
),
ffn=StageEntry(
prepare=into_ffn.prepare,
@@ -250,7 +250,6 @@ class CuteDSLFusion:
self._mlp_input_reduce_output_and_update_and_read_residual,
layer,
),
may_return_new_residual=True,
),
)
return ()
+98 -29
View File
@@ -59,9 +59,11 @@ from sglang.srt.layers.communicator.layout import (
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
_all_reduce_then_to_local_tokens,
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
_mlp_input_without_dp,
_redistribute_output,
_reduce_and_redistribute_output_step,
_to_local_tokens,
@@ -79,6 +81,7 @@ from sglang.srt.layers.communicator.residual.access import (
finish_layer_stack,
fold,
from_pp,
norm_output,
snapshot,
written,
)
@@ -94,6 +97,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
@@ -244,6 +248,7 @@ class LayerCommunicator:
add_to_output = staticmethod(add_to_output)
buffer = staticmethod(buffer)
fold = staticmethod(fold)
norm_output = staticmethod(norm_output)
written = staticmethod(written)
finish_layer_stack = staticmethod(finish_layer_stack)
@@ -361,6 +366,46 @@ class LayerCommunicator:
else None
)
# Bind aux ownership to the selected input implementation once.
for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"):
steps = self.__dict__[name]
if steps is None or steps.ffn is None:
continue
prepare = steps.ffn.prepare
if isinstance(prepare, partial):
prepare = prepare.keywords.get("step", prepare)
norm = self.post_attention_layernorm
known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type(
norm
).forward_with_allreduce_fusion in (
RMSNorm.forward_with_allreduce_fusion,
GemmaRMSNorm.forward_with_allreduce_fusion,
)
if (
known_norm
and isinstance(prepare, partial)
and prepare.func is _mlp_input_without_dp
and not prepare.keywords["gathers_residual"]
and steps.ffn.fused
and steps.ffn.fused[0].run
== self._mlp_input_reduce_output_and_update_and_read_residual
):
self.__dict__[name] = msgspec.structs.replace(
steps,
attention=msgspec.structs.replace(
steps.attention,
capture_preserves_residual=self._capture_keeps_residual,
),
)
def _capture_keeps_residual(self, value, forward_batch):
# The CUDA wrapper allocates a fresh residual even on backend fallback.
return (
not _use_aiter
and get_parallel().attn_tp_size > 1
and apply_flashinfer_allreduce_fusion(value.shape[0])
)
def _init_stage(self, stage: "LayerStage") -> None:
"""A layer that is one stage: every batch runs the two boundaries its
declarations give."""
@@ -387,6 +432,7 @@ class LayerCommunicator:
input_move=into.input_move,
handoff=None if is_ffn else _hand_qkv_hook_its_input,
fused=into.fused,
capture_move=into.capture_move,
)
self._steps = BoundarySteps(
attention=None if is_ffn else entry,
@@ -643,7 +689,38 @@ class LayerCommunicator:
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
post_residual_addition: Optional[torch.Tensor] = None,
quant_format: str = "",
capture_output: Optional[Callable] = None,
):
# Aux consumers need a materialized output before the input norm.
# Complete communication here, preserving the existing add+norm kernel
# and its FP32 accumulation. Its residual result also supplies capture.
capture_before_read = capture_output is not None and (
residual is None
or (
isinstance(residual, ResidualStream)
and residual.residual is None
and residual.pending is None
)
or post_residual_addition is not None
)
if capture_output is not None:
hidden_states = (
residual.complete(hidden_states)
if isinstance(residual, ResidualStream)
else reduce_output(hidden_states)
)
if capture_before_read:
# Embeddings precede enter; HF deepstack capture precedes the
# extra addition. Neither is the residual returned by the read.
value, previous = (
residual.finish(hidden_states)
if isinstance(residual, ResidualStream)
else (hidden_states, residual)
)
if previous is None:
capture_output(value)
else:
capture_output(value + previous, owned=True)
hidden_states, residual = self.prepare_attn(
hidden_states,
residual,
@@ -651,6 +728,17 @@ class LayerCommunicator:
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
if capture_output is not None and not capture_before_read:
value = residual.residual
move = self.attn.entry(forward_batch).capture_move
if move is not None:
value = move(value, forward_batch=forward_batch)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
capture_output(
value,
owned=self.attn.entry(forward_batch).capture_move_allocates
or (move is None and keeps is not None and keeps(value, forward_batch)),
)
if captured_last_layer_outputs is not None:
residual_value = residual.residual
move = self.attn.entry(forward_batch).input_move
@@ -663,36 +751,18 @@ class LayerCommunicator:
context=self._context,
)
)
if (
gathered_last_layer_output is residual_value
# An accumulator that copies on append already holds a snapshot.
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
and not self._post_attn_residual_is_read_only(
residual_value, forward_batch
)
):
gathered_last_layer_output = residual_value.clone()
captured_last_layer_outputs.append(gathered_last_layer_output)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
captured_last_layer_outputs.capture(
gathered_last_layer_output,
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
or (
move is None
and keeps is not None
and keeps(residual_value, forward_batch)
),
)
return hidden_states, residual
def _post_attn_residual_is_read_only(
self, residual: torch.Tensor, forward_batch: ForwardBatch
) -> bool:
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
Of the base fused entry's kernels only flashinfer's writes a fresh
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
kernel and every plain norm fold into ``residual`` in place. It runs when
``prepare_mlp`` selected a fused entry that may return a new residual and
the batch is not input-scattered.
"""
return (
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
def publish_attn_lora_layout(self) -> None:
"""Attention consumes the DP-local token batch."""
if self._publish_lora_layout:
@@ -829,7 +899,6 @@ class LayerCommunicator:
FusedMlpInput(
completes=SumGroup.ATTN_TP,
run=self._mlp_input_reduce_output_and_update_and_read_residual,
may_return_new_residual=True,
),
)
@@ -100,6 +100,23 @@ def finish_layer_stack(
return reduce_output(hidden_states), residual
def norm_output(hidden_states, residual, norm, capture_output=None):
"""The final norm and an optional capture of the same updated residual.
Keep add and norm together: normalizing a separately rounded residual is
not numerically equivalent to the fused kernel's FP32 accumulation. The
capture callback owns retention of the borrowed residual storage.
"""
if residual is None:
if capture_output is not None:
capture_output(hidden_states)
return norm(hidden_states)
hidden_states, residual = norm(hidden_states, residual)
if capture_output is not None:
capture_output(residual)
return hidden_states
def from_pp(
tensors: PPProxyTensors,
*,
@@ -235,8 +235,8 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
tensor, output_buffer.tensors[key], num_tokens
)
return
if isinstance(output, (list, tuple)) and isinstance(
output_buffer, type(output)
if (isinstance(output, list) and isinstance(output_buffer, list)) or (
isinstance(output, tuple) and isinstance(output_buffer, tuple)
):
if len(output) != len(output_buffer):
raise ValueError(
+11 -9
View File
@@ -31,6 +31,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -722,7 +726,8 @@ class BailingMoEBlock(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
@@ -730,6 +735,7 @@ class BailingMoEBlock(nn.Module):
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
)
)
@@ -822,16 +828,9 @@ class BailingMoEModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -843,6 +842,9 @@ class BailingMoEModel(nn.Module):
if getattr(layer, "_is_layer_to_capture", False)
else None
),
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
+29 -13
View File
@@ -28,6 +28,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1178,10 +1179,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if not forward_batch.forward_mode.is_idle():
@@ -1370,7 +1377,11 @@ class BailingMoELinearModel(nn.Module):
and capture_mode.need_capture()
)
if capture_aux:
dspark_aux_hidden_states: List[torch.Tensor] = []
dspark_aux_hidden_states = AuxHiddenStateList()
def capture_output(value, *, owned=False):
if value.shape[0] != 0:
dspark_aux_hidden_states.capture(value, owned=owned)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -1381,13 +1392,12 @@ class BailingMoELinearModel(nn.Module):
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
capture_output=capture_output
if capture_aux
and i > self.start_layer
and i - 1 in self.layers_to_capture
else None,
)
if capture_aux and i in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, skip_empty=True
)
if snapshot is not None:
dspark_aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
@@ -1399,10 +1409,16 @@ class BailingMoELinearModel(nn.Module):
)
else:
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
capture_output
if capture_aux and self.end_layer - 1 in self.layers_to_capture
else None
),
)
if capture_aux and len(dspark_aux_hidden_states) > 0:
return hidden_states, dspark_aux_hidden_states
return hidden_states
+14 -13
View File
@@ -38,6 +38,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -936,13 +937,17 @@ class Glm4MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=self.attn_quant_format,
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -1103,22 +1108,18 @@ class Glm4MoeModel(nn.Module):
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+14 -13
View File
@@ -32,6 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -625,12 +626,16 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
getattr(self, "_gfx95_quant_format", ""),
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
quant_format=getattr(self, "_gfx95_quant_format", ""),
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -800,16 +805,9 @@ class Glm4MoeLiteModel(nn.Module):
normal_end_layer = self.first_k_dense_replace
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -817,6 +815,9 @@ class Glm4MoeLiteModel(nn.Module):
forward_batch,
residual,
zero_allocator,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+22 -20
View File
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_distribution import (
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -918,13 +919,17 @@ class Glm5NextDecoderLayer(nn.Module):
zero_allocator: Optional[BumpAllocator] = None,
gemm_output_zero_allocator: BumpAllocator = None,
prev_topk_indices: Optional[torch.Tensor] = None,
capture_output=None,
):
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
hidden_states = self.self_attn(
@@ -1139,7 +1144,7 @@ class Glm5NextModel(nn.Module):
normal_end_layer = self.first_k_dense_replace
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
topk_indices = None
for i in range(normal_start_layer, normal_end_layer):
# NOTE: torch dynamo does not support graph break in context manager
@@ -1149,22 +1154,16 @@ class Glm5NextModel(nn.Module):
else get_global_expert_distribution_recorder().with_current_layer(i)
)
with ctx:
if i in self.layers_to_capture:
if self.dflash_capture and self.config.mhc:
# hc_post has completed the streams; contracting them
# creates the retained value without copying the wide input.
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
else:
hidden_states, aux_hidden_state = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
def capture_output(aux_hidden_state, *, owned=False):
aux_hidden_state = self._prepare_aux_hidden_state(aux_hidden_state)
owned = owned or (self.dflash_capture and self.config.mhc)
if self.enable_a2a_moe and i > self.first_k_dense_replace:
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
aux_hidden_state, dim=0
)
aux_hidden_states.append(aux_hidden_state)
group = get_parallel().attn_tp_group
aux_hidden_state = group.all_gather(aux_hidden_state, dim=0)
owned = owned or group.world_size > 1
aux_hidden_states.capture(aux_hidden_state, owned=owned)
layer = self.layers[i]
hidden_states, residual, topk_indices = layer(
positions,
@@ -1174,6 +1173,9 @@ class Glm5NextModel(nn.Module):
zero_allocator,
gemm_output_zero_allocator,
prev_topk_indices=topk_indices,
capture_output=capture_output
if i in self.layers_to_capture
else None,
)
if normal_end_layer != self.end_layer:
+31 -19
View File
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -618,9 +619,15 @@ class GptOssDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if hidden_states.shape[0] != 0:
@@ -710,24 +717,19 @@ class GptOssModel(nn.Module):
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
aux_hidden_states = []
if self.start_layer in self.layers_to_capture:
aux_hidden_states.append(
self.layers[self.start_layer].layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if i + 1 in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual
)
aux_hidden_states.append(snapshot)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
@@ -741,10 +743,20 @@ class GptOssModel(nn.Module):
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
)
elif self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if len(aux_hidden_states) == 0:
return hidden_states
+6 -2
View File
@@ -10,6 +10,10 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
)
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -644,7 +648,7 @@ class InternS2MobiusAttentionDecoderLayer(
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
meta_mlp: nn.ModuleList,
captured_last_layer_outputs: list[torch.Tensor] | None = None,
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
**kwargs,
):
del kwargs
@@ -764,7 +768,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(self.layers):
hidden_states, residual = layer(
positions=positions,
+27 -19
View File
@@ -19,6 +19,7 @@ from torch import nn
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -475,9 +476,15 @@ class LagunaDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -589,18 +596,17 @@ class LagunaModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
@@ -613,14 +619,16 @@ class LagunaModel(nn.Module):
)
if hidden_states.shape[0] != 0:
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states,
residual,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
)
if len(aux_hidden_states) == 0:
return hidden_states
return hidden_states, aux_hidden_states
+7 -3
View File
@@ -18,7 +18,7 @@
import logging
from contextlib import nullcontext
from functools import lru_cache
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
import torch
import triton
@@ -30,6 +30,10 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -997,7 +1001,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states, residual = (
@@ -1149,7 +1153,7 @@ class MiniMaxM2Model(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+6 -2
View File
@@ -35,6 +35,10 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1369,7 +1373,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> torch.Tensor:
hidden_states, residual = (
@@ -1484,7 +1488,7 @@ class MiniMaxM3Model(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+6 -2
View File
@@ -40,6 +40,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1070,7 +1074,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -1183,7 +1187,7 @@ class Qwen2MoeModel(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
if forward_batch.can_run_tbo:
hidden_states, residual = model_forward_maybe_tbo(
layers=self.layers,
+8 -3
View File
@@ -23,7 +23,6 @@ import torch
import torch.nn as nn
import triton
# Layers - Attention
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
fused_qkvzba_split_reshape_cat_contiguous,
@@ -41,6 +40,12 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
# Layers - Attention
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
@@ -1540,7 +1545,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
hidden_states, residual = (
@@ -1825,7 +1830,7 @@ class Qwen3_5ForCausalLM(nn.Module):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
# Pass through decoder layers
for layer_idx in range(self.start_layer, self.end_layer):
layer = self.layers[layer_idx]
+4 -1
View File
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
@@ -882,7 +883,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -892,6 +894,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
)
+7 -3
View File
@@ -15,6 +15,10 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -550,7 +554,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs,
):
forward_batch = kwargs.get("forward_batch", None)
@@ -826,7 +830,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
**kwargs: Any,
):
hidden_states, residual = (
@@ -931,7 +935,7 @@ class Qwen3NextModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(len(self.layers)):
layer = self.layers[i]
with get_global_expert_distribution_recorder().with_current_layer(i):
+8 -6
View File
@@ -24,6 +24,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -102,16 +103,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer]
):
layer_idx += self.start_layer
if layer_idx in self.layers_to_capture:
hidden_states, snapshot = layer.layer_communicator.capture_output(
hidden_states, residual, at_input=True
)
aux_hidden_states.append(snapshot)
if self.use_hf_deepstack_order:
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
@@ -126,6 +122,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
forward_batch,
residual,
post_residual_addition=deepstack_embeds,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
)
else:
# Inference path: add deepstack directly to hidden_states at the end of the layer
@@ -135,6 +134,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None,
)
if (
input_deepstack_embeds is not None
+2 -1
View File
@@ -21,6 +21,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather,
@@ -1748,7 +1749,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
if self.has_ple
else None
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
if i + 1 < self.end_layer:
@@ -74,6 +74,7 @@ patches:
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
)
@@ -131,6 +132,7 @@ patches:
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
**kwargs,
)
)
@@ -172,7 +172,6 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel():
assert fusions[1].run == base
# The workspace reduces over the TP group, which is the attention-TP group here.
assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2
assert all(f.may_return_new_residual for f in fusions)
# A residual on each rank's slice is gathered first, which the
# workspace does not do.
comm._declared = SimpleNamespace(
@@ -0,0 +1,204 @@
"""Aux owns retained storage, including captures from reusable gather buffers."""
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker
from sglang.srt.layers.communicator import LayerCommunicator as BoundaryPlan
from sglang.srt.layers.communicator.residual.access import norm_output
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestAuxStorage(CustomTestCase):
def test_constructed_layer_binds_read_only_residual_capture(self):
import test_declared_decoder_boundary as fixture
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import layernorm
from sglang.test.communicator_patch import patch_communicator
parallel = fixture.parallel_of(attn_dp=1, attn_tp=2)
with (
fixture.planning(parallel),
patch_communicator("_use_aiter", False),
patch_communicator("apply_flashinfer_allreduce_fusion", return_value=True),
):
layer = comm.LayerCommunicator(
layer_facts=fixture.layer_facts(1, 3),
input_layernorm=layernorm.RMSNorm(4),
post_attention_layernorm=layernorm.RMSNorm(4),
)
keeps = layer._steps.attention.capture_preserves_residual
self.assertIsNotNone(keeps)
self.assertTrue(keeps(torch.ones(2, 4), SimpleNamespace()))
def test_breakable_graph_copies_aux_collector_into_plain_list(self):
from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import (
BreakableCudaGraphBackend,
)
backend = BreakableCudaGraphBackend.__new__(BreakableCudaGraphBackend)
source = AuxHiddenStateList([torch.full((4, 3), 7.0)])
buffers = [torch.zeros(4, 3)]
backend._copy_output_to_buffer(
(torch.ones(4, 3), source), (torch.zeros(4, 3), buffers), 2
)
torch.testing.assert_close(buffers[0][:2], torch.full((2, 3), 7.0))
self.assertEqual(buffers[0][2:].count_nonzero(), 0)
with self.assertRaises(TypeError):
backend._copy_output_to_buffer(source, tuple(buffers), 2)
def test_capture_move_owns_gather_but_not_slice(self):
from sglang.srt.layers.communicator.boundary import Boundary
from sglang.srt.layers.communicator.layout import Layout, TokenAxis
from sglang.test.communicator_patch import patch_communicator
full = Layout(frozenset())
sharded = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
source = torch.ones(2, 3)
for rows, target, owns in ((sharded, full, True), (full, sharded, False)):
edge = SimpleNamespace(
residual_to=rows, produced=SimpleNamespace(layout=target)
)
boundary = Boundary(edge=edge, prepare=None)
self.assertEqual(boundary.capture_move_allocates, owns)
with (
patch_communicator(
"_redistribute_from_attn_tp_shards",
side_effect=lambda x: torch.cat((x, x)),
),
patch_communicator(
"get_parallel",
return_value=SimpleNamespace(attn_tp_size=2, attn_tp_rank=0),
),
):
value = boundary.capture_move(source, forward_batch=None)
outputs = AuxHiddenStateList()
if owns:
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
outputs.capture(value, owned=boundary.capture_move_allocates)
self.assertIs(outputs[0], value)
else:
outputs.capture(value, owned=boundary.capture_move_allocates)
self.assertIsNot(outputs[0], value)
def test_list_snapshots_borrowed_views_and_reused_buffers(self):
outputs = AuxHiddenStateList()
buffer = torch.full((4, 3), 2.0)
outputs.capture(buffer[:2])
buffer.fill_(5)
outputs.capture(buffer[:2])
buffer.zero_()
torch.testing.assert_close(outputs[0], torch.full((2, 3), 2.0))
torch.testing.assert_close(outputs[1], torch.full((2, 3), 5.0))
def test_list_adopts_owned_output_without_copying(self):
outputs = AuxHiddenStateList()
value = torch.ones(2, 3)
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
outputs.capture(value, owned=True)
self.assertIs(outputs[0], value)
def boundary(self, stream, *, move=None):
boundary = BoundaryPlan.__new__(BoundaryPlan)
boundary.input_layernorm = None
boundary._context = None
boundary.prepare_attn = Mock(return_value=(stream.residual, stream))
boundary.attn = SimpleNamespace(
entry=lambda _: SimpleNamespace(
input_move=move,
capture_move=None,
capture_move_allocates=False,
capture_preserves_residual=None,
)
)
return boundary
def test_packer_receives_borrowed_boundary_value_without_intermediate_clone(self):
for callback in (False, True):
with self.subTest(callback=callback):
value = torch.full((2, 3), 4.0)
stream = ResidualStream(value)
stage = self.boundary(stream)
outputs = AuxHiddenStatePacker(1)
kwargs = (
{"capture_output": outputs.capture}
if callback
else {"captured_last_layer_outputs": outputs}
)
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
stage.prepare_attn_and_capture_last_layer_outputs(
value, stream, None, **kwargs
)
value.zero_()
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
def test_gather_storage_is_borrowed_even_if_it_is_a_different_tensor(self):
source = torch.ones(2, 3)
gathered = torch.full((4, 3), 7.0)
stream = ResidualStream(source)
move = Mock(return_value=gathered)
stage = self.boundary(stream, move=move)
outputs = AuxHiddenStateList()
stage.prepare_attn_and_capture_last_layer_outputs(source, stream, None, outputs)
gathered.zero_()
source.zero_()
torch.testing.assert_close(outputs[0], torch.full((4, 3), 7.0))
move.assert_called_once()
def test_embedding_and_final_norm_capture_write_directly_to_packer(self):
for residual in (None, torch.ones(2, 3)):
with self.subTest(residual=residual):
hidden = torch.full((2, 3), 4.0)
outputs = AuxHiddenStatePacker(1)
updated = torch.full((2, 3), 5.0)
norm = Mock(
return_value=hidden if residual is None else (hidden, updated)
)
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
norm_output(hidden, residual, norm, outputs.capture)
hidden.zero_()
updated.zero_()
torch.testing.assert_close(
outputs.finalize(),
torch.full((2, 3), 4.0 if residual is None else 5.0),
)
def test_initial_capture_precedes_in_place_prepare_without_intermediate_clone(self):
hidden = torch.full((2, 3), 4.0)
stream = ResidualStream()
outputs = AuxHiddenStatePacker(1)
stage = self.boundary(stream)
def prepare(value, stream, batch, **kwargs):
value.zero_()
stream.write(value)
return value, stream
stage.prepare_attn = prepare
with patch.object(
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
):
stage.prepare_attn_and_capture_last_layer_outputs(
hidden, stream, None, capture_output=outputs.capture
)
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
if __name__ == "__main__":
unittest.main()
@@ -261,11 +261,9 @@ class TestNonAlternatingEdges(CustomTestCase):
residual=full,
residual_to=full,
)
over_tp = comm.FusedMlpInput(
completes=SumGroup.TP, run=MagicMock(), may_return_new_residual=False
)
over_tp = comm.FusedMlpInput(completes=SumGroup.TP, run=MagicMock())
over_attention_tp = comm.FusedMlpInput(
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False
completes=SumGroup.ATTN_TP, run=MagicMock()
)
boundary = make_boundary(edge, fusions=(over_tp, over_attention_tp))
self.assertEqual(boundary.fused, (over_tp,))
@@ -370,9 +368,7 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
def test_fused_kernels_take_only_a_plain_add(self):
full = rows(sizes(tp=2))
fused = comm.FusedMlpInput(
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False
)
fused = comm.FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock())
carried = MagicMock()
for update, offered in ((comm.ADD, True), (_WrittenIn(), False)):
with self.subTest(adds_plainly=offered):
@@ -569,9 +565,7 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase):
torch.testing.assert_close(out, expected + 100)
def test_a_fused_kernel_runs_only_for_a_read_that_leaves_the_residual(self):
fused = FusedMlpInput(
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=True
)
fused = FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock())
for norms_plainly in (True, False):
with self.subTest(norms_plainly=norms_plainly):
step, tried = self._ffn_input(
@@ -276,9 +276,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
(entry,) if fuses else (),
)
self.assertIs(
any(
f.may_return_new_residual for f in communicator._steps.ffn.fused
),
bool(communicator._steps.ffn.fused),
fuses,
)
self.assertFalse(communicator._steps.returns_over_dp)
@@ -1095,11 +1093,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
norm only where those are the steps, and only if it completes the sum the
attention output owes."""
def fused(self, completes, may_return_new_residual):
def fused(self, completes):
return comm.FusedMlpInput(
completes=completes,
run=lambda h, r, fb: None,
may_return_new_residual=may_return_new_residual,
)
def select(self, *, attn_dp, fusions):
@@ -1125,45 +1122,18 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
return steps, fused
def test_a_kernel_over_another_group_is_not_chosen(self):
over_tp = self.fused(SumGroup.TP, True)
over_attention_tp = self.fused(SumGroup.ATTN_TP, False)
over_tp = self.fused(SumGroup.TP)
over_attention_tp = self.fused(SumGroup.ATTN_TP)
steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp))
self.assertIs(steps.func, comm_ops._mlp_input_without_dp)
self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,))
self.assertEqual(chosen, (over_attention_tp,))
def test_no_kernel_is_chosen_where_rows_are_gathered(self):
steps, chosen = self.select(
attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP, True),)
)
steps, chosen = self.select(attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP),))
self.assertIs(steps.func, comm_ops._mlp_input_dp_partial)
self.assertEqual(chosen, ())
def test_aux_capture_follows_the_chosen_kernels(self):
for kernels, expected in (
((self.fused(SumGroup.ATTN_TP, False),), False),
((self.fused(SumGroup.ATTN_TP, True),), True),
((self.fused(SumGroup.TP, True),), False),
):
with self.subTest(kernels=kernels):
class Declaring(LayerCommunicator):
def _select_mlp_input_fusions(self):
return kernels
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = Declaring(
layer_facts=layer_facts(1, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertIs(
any(
f.may_return_new_residual for f in communicator._steps.ffn.fused
),
expected,
)
class TestTheSequenceParallelRegion(CustomTestCase):
"""While a LayerNorm SP region is active, the layer runs the steps its
@@ -1492,7 +1462,6 @@ class TestOneRepresentation(CustomTestCase):
"_ffn_output",
"_postprocess_scatters_to_local_tokens",
"_ffn_sum_is_movable",
"_mlp_input_may_return_new_residual",
)
def test_the_layer_holds_steps_only(self):
@@ -4,9 +4,11 @@ from unittest.mock import Mock
import torch
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
BoundarySteps,
EdgeDecl,
LayerCommunicator,
Layout,
StageEntry,
StageInput,
@@ -162,6 +164,107 @@ class TestResidualStream(CustomTestCase):
with self.assertRaises(RuntimeError):
self.stream.input(hidden.clone())
def test_capture_and_compute_share_the_fused_read(self):
boundary = LayerCommunicator.__new__(LayerCommunicator)
events = []
outputs = AuxHiddenStateList()
def prepare(hidden, stream, batch, **kwargs):
events.append("add_norm")
self.assertIsNone(stream.pending.owed)
# A fused read returns both the norm output and its updated residual.
updated = hidden + stream.residual
stream.write(updated)
return updated * 3, stream
def capture(value, *, owned=False):
events.append("capture")
outputs.capture(value, owned=owned)
boundary.prepare_attn = Mock(side_effect=prepare)
boundary.attn = SimpleNamespace(
entry=lambda batch: SimpleNamespace(
capture_move=None,
capture_move_allocates=False,
capture_preserves_residual=None,
)
)
output, stream = boundary.prepare_attn_and_capture_last_layer_outputs(
self.hidden, self.stream, None, capture_output=capture
)
self.assertEqual(events, ["add_norm", "capture"])
self.group.all_reduce.assert_called_once()
boundary.prepare_attn.assert_called_once()
torch.testing.assert_close(output, torch.full((2, 4), 15.0))
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
stream.residual.zero_()
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
def test_deepstack_capture_precedes_the_extra_addition(self):
boundary = LayerCommunicator.__new__(LayerCommunicator)
extra = torch.full_like(self.partial, 7.0)
outputs = AuxHiddenStateList()
def prepare(hidden, stream, batch, **kwargs):
self.assertEqual(len(outputs), 1)
self.assertIs(kwargs["post_residual_addition"], extra)
updated = hidden + stream.residual + extra
stream.write(updated)
return updated * 3, stream
boundary.prepare_attn = Mock(side_effect=prepare)
boundary.attn = SimpleNamespace(
entry=lambda batch: SimpleNamespace(
capture_move=None,
capture_move_allocates=False,
capture_preserves_residual=None,
)
)
output, _ = boundary.prepare_attn_and_capture_last_layer_outputs(
self.hidden,
self.stream,
None,
post_residual_addition=extra,
capture_output=outputs.capture,
)
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
torch.testing.assert_close(output, torch.full((2, 4), 36.0))
self.group.all_reduce.assert_called_once()
def test_capture_restores_the_producers_rows_after_a_scattered_read(self):
from unittest.mock import patch
from sglang.srt.layers.communicator import TokenAxis
full = Layout(frozenset())
local = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
boundary = make_boundary(
EdgeDecl(StageOutput(full), StageInput(local), full, local)
)
shard = torch.full((2, 4), 3.0)
expected = torch.cat([shard, torch.full_like(shard, 7.0)])
with patch(
"sglang.srt.layers.communicator.ops._redistribute_from_attn_tp_shards",
return_value=expected,
) as gather:
captured = boundary.capture_move(shard, forward_batch=None)
gather.assert_called_once_with(shard)
torch.testing.assert_close(captured, expected)
torch.testing.assert_close(shard, torch.full((2, 4), 3.0))
def test_final_capture_uses_the_norms_residual_result(self):
from sglang.srt.layers.communicator.residual.access import norm_output
outputs = AuxHiddenStateList()
updated = torch.full_like(self.partial, 5.0)
normalized = torch.full_like(self.partial, 13.0)
norm = Mock(return_value=(normalized, updated))
result = norm_output(self.partial, self.residual, norm, outputs.capture)
self.assertIs(result, normalized)
norm.assert_called_once_with(self.partial, self.residual)
updated.zero_()
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
def test_deepstack_adds_once_after_completing_the_sum(self):
hidden, stream = add_to_output(
self.hidden, self.stream, torch.full((2, 4), 7.0)
@@ -118,6 +118,9 @@ class DeferringLayer(nn.Module):
# later norms mutate the residual; captured snapshots must stay intact
residual.add_(hidden_states)
stream.write(residual)
capture = kwargs.get("capture_output")
if capture is not None:
capture(residual.clone())
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
partial = torch.full_like(hidden_states, 0.5)
hidden_states, residual = ffn_exit.finish(partial, stream)
@@ -130,7 +133,7 @@ class SumNorm(nn.Module):
def forward(self, hidden_states, residual=None, **kwargs):
if residual is None:
return hidden_states
return hidden_states + residual, residual
return hidden_states + residual, hidden_states + residual
def build_model(model_cls, *, defer, capture):
@@ -211,6 +214,29 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase):
reduce.call_count, NUM_LAYERS - 1 if defer else 0
)
def test_terminal_capture_uses_the_final_norm_boundary(self):
inputs = torch.ones(2, 4)
batch = SimpleNamespace(
can_run_tbo=False,
forward_mode=ForwardMode.DECODE,
capture_hidden_mode=SimpleNamespace(need_capture=lambda: True),
)
for cls in (GptOssModel, LagunaModel, BailingMoELinearModel):
with self.subTest(model=cls.__name__):
model = build_model(cls, defer=True, capture=True)
model.layers_to_capture = [
NUM_LAYERS - 1 if cls is BailingMoELinearModel else NUM_LAYERS
]
result, snapshots = model(
input_ids=None,
positions=None,
forward_batch=batch,
input_embeds=inputs.clone(),
)
self.assertEqual(len(snapshots), 1)
torch.testing.assert_close(snapshots[0], inputs + NUM_LAYERS)
torch.testing.assert_close(result, inputs + NUM_LAYERS)
class TestPipelineResidualReception(CustomTestCase):
def test_models_keep_the_received_residual_contribution(self):
@@ -238,6 +264,29 @@ class TestPipelineResidualReception(CustomTestCase):
)
torch.testing.assert_close(result, initial_residual + NUM_LAYERS)
def test_bailing_capture_does_not_include_a_previous_pipeline_ranks_layer(self):
model = build_model(BailingMoELinearModel, defer=True, capture=True)
model.pp_group.is_first_rank = False
model.start_layer = 2
model.layers_to_capture = [model.start_layer - 1]
inputs = torch.zeros(2, 4)
residual = torch.full_like(inputs, 0.25)
batch = SimpleNamespace(
can_run_tbo=False,
forward_mode=ForwardMode.DECODE,
capture_hidden_mode=SimpleNamespace(need_capture=lambda: True),
)
result = model(
None,
None,
batch,
pp_proxy_tensors=PPProxyTensors(
{"hidden_states": inputs, "residual": residual}
),
)
self.assertIsInstance(result, torch.Tensor)
torch.testing.assert_close(result, torch.full_like(inputs, 2.25))
def test_written_streams_do_not_read_a_separate_residual(self):
comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
comm_instance._residual = SimpleNamespace(