[Refactor] Capture auxiliary states at residual reads (#41550)
This commit is contained in:
@@ -9,6 +9,18 @@ import torch
|
||||
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
|
||||
|
||||
|
||||
class AuxHiddenStateList(list):
|
||||
"""Retain aux snapshots independently of storage reused by later layers."""
|
||||
|
||||
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
|
||||
"""Copy borrowed storage, or adopt a value whose ownership is transferred.
|
||||
|
||||
Views and reusable communication buffers are borrowed even when they
|
||||
are different tensor objects from the source.
|
||||
"""
|
||||
self.append(hidden if owned else hidden.clone())
|
||||
|
||||
|
||||
class AuxHiddenStatePacker:
|
||||
"""Drop-in for the ``[]`` a model collects Eagle3/DFlash captures into.
|
||||
|
||||
@@ -17,9 +29,6 @@ class AuxHiddenStatePacker:
|
||||
Assumes all captures share leading shape and feature size.
|
||||
"""
|
||||
|
||||
# ``append`` copies, so producers need not clone a tensor they later mutate.
|
||||
copies_on_append = True
|
||||
|
||||
def __init__(self, num_captures: int) -> None:
|
||||
self._num_captures = int(num_captures)
|
||||
self._buffer: Optional[torch.Tensor] = None
|
||||
@@ -37,6 +46,10 @@ class AuxHiddenStatePacker:
|
||||
self._buffer[..., start : start + self._feature_size].copy_(hidden)
|
||||
self._idx += 1
|
||||
|
||||
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
|
||||
"""Write directly to the final packed buffer, without an intermediate copy."""
|
||||
self.append(hidden)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._idx
|
||||
|
||||
@@ -49,8 +62,8 @@ class AuxHiddenStatePacker:
|
||||
return self._buffer
|
||||
|
||||
|
||||
# What a model hands down the capture path: a plain list, or a packer writing in place.
|
||||
AuxHiddenStateAccumulator = Union[List[torch.Tensor], AuxHiddenStatePacker]
|
||||
# Both collectors accept borrowed values through capture; storage belongs to aux.
|
||||
AuxHiddenStateAccumulator = Union[AuxHiddenStateList, AuxHiddenStatePacker]
|
||||
|
||||
|
||||
def pack_aux_hidden_states(aux_hidden_states: AuxHiddenStates) -> torch.Tensor:
|
||||
|
||||
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
_mlp_input_slice,
|
||||
_mlp_input_without_dp,
|
||||
_read_input,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
@@ -479,8 +480,6 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
# It may hand back a new residual and leave the one it took unchanged.
|
||||
may_return_new_residual: bool
|
||||
|
||||
|
||||
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
@@ -548,6 +547,10 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
handoff: Optional[Callable] = None
|
||||
# The fused kernels prepare tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Return the updated residual to the producer's rows for aux capture.
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
@@ -624,6 +627,19 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
|
||||
)
|
||||
|
||||
@property
|
||||
def capture_move_allocates(self) -> bool:
|
||||
"""Returning through a gather allocates; returning through a cut aliases."""
|
||||
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
|
||||
|
||||
@property
|
||||
def capture_move(self) -> Optional[Callable]:
|
||||
if self.edge.residual_to == self.edge.produced.layout:
|
||||
return None
|
||||
return partial(
|
||||
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
|
||||
)
|
||||
|
||||
|
||||
def make_boundary(
|
||||
edge: EdgeDecl,
|
||||
@@ -808,6 +824,8 @@ def _select_boundary_steps(
|
||||
input_rows=into_attention.input_rows,
|
||||
input_move=into_attention.input_move,
|
||||
handoff=attention_handoff,
|
||||
capture_move=into_attention.capture_move,
|
||||
capture_move_allocates=into_attention.capture_move_allocates,
|
||||
),
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
|
||||
@@ -250,7 +250,6 @@ class CuteDSLFusion:
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
layer,
|
||||
),
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
return ()
|
||||
|
||||
@@ -59,9 +59,11 @@ from sglang.srt.layers.communicator.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
_all_reduce_then_to_local_tokens,
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
_mlp_input_without_dp,
|
||||
_redistribute_output,
|
||||
_reduce_and_redistribute_output_step,
|
||||
_to_local_tokens,
|
||||
@@ -79,6 +81,7 @@ from sglang.srt.layers.communicator.residual.access import (
|
||||
finish_layer_stack,
|
||||
fold,
|
||||
from_pp,
|
||||
norm_output,
|
||||
snapshot,
|
||||
written,
|
||||
)
|
||||
@@ -94,6 +97,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
@@ -244,6 +248,7 @@ class LayerCommunicator:
|
||||
add_to_output = staticmethod(add_to_output)
|
||||
buffer = staticmethod(buffer)
|
||||
fold = staticmethod(fold)
|
||||
norm_output = staticmethod(norm_output)
|
||||
written = staticmethod(written)
|
||||
finish_layer_stack = staticmethod(finish_layer_stack)
|
||||
|
||||
@@ -361,6 +366,46 @@ class LayerCommunicator:
|
||||
else None
|
||||
)
|
||||
|
||||
# Bind aux ownership to the selected input implementation once.
|
||||
for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"):
|
||||
steps = self.__dict__[name]
|
||||
if steps is None or steps.ffn is None:
|
||||
continue
|
||||
prepare = steps.ffn.prepare
|
||||
if isinstance(prepare, partial):
|
||||
prepare = prepare.keywords.get("step", prepare)
|
||||
norm = self.post_attention_layernorm
|
||||
known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type(
|
||||
norm
|
||||
).forward_with_allreduce_fusion in (
|
||||
RMSNorm.forward_with_allreduce_fusion,
|
||||
GemmaRMSNorm.forward_with_allreduce_fusion,
|
||||
)
|
||||
if (
|
||||
known_norm
|
||||
and isinstance(prepare, partial)
|
||||
and prepare.func is _mlp_input_without_dp
|
||||
and not prepare.keywords["gathers_residual"]
|
||||
and steps.ffn.fused
|
||||
and steps.ffn.fused[0].run
|
||||
== self._mlp_input_reduce_output_and_update_and_read_residual
|
||||
):
|
||||
self.__dict__[name] = msgspec.structs.replace(
|
||||
steps,
|
||||
attention=msgspec.structs.replace(
|
||||
steps.attention,
|
||||
capture_preserves_residual=self._capture_keeps_residual,
|
||||
),
|
||||
)
|
||||
|
||||
def _capture_keeps_residual(self, value, forward_batch):
|
||||
# The CUDA wrapper allocates a fresh residual even on backend fallback.
|
||||
return (
|
||||
not _use_aiter
|
||||
and get_parallel().attn_tp_size > 1
|
||||
and apply_flashinfer_allreduce_fusion(value.shape[0])
|
||||
)
|
||||
|
||||
def _init_stage(self, stage: "LayerStage") -> None:
|
||||
"""A layer that is one stage: every batch runs the two boundaries its
|
||||
declarations give."""
|
||||
@@ -387,6 +432,7 @@ class LayerCommunicator:
|
||||
input_move=into.input_move,
|
||||
handoff=None if is_ffn else _hand_qkv_hook_its_input,
|
||||
fused=into.fused,
|
||||
capture_move=into.capture_move,
|
||||
)
|
||||
self._steps = BoundarySteps(
|
||||
attention=None if is_ffn else entry,
|
||||
@@ -643,7 +689,38 @@ class LayerCommunicator:
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
quant_format: str = "",
|
||||
capture_output: Optional[Callable] = None,
|
||||
):
|
||||
# Aux consumers need a materialized output before the input norm.
|
||||
# Complete communication here, preserving the existing add+norm kernel
|
||||
# and its FP32 accumulation. Its residual result also supplies capture.
|
||||
capture_before_read = capture_output is not None and (
|
||||
residual is None
|
||||
or (
|
||||
isinstance(residual, ResidualStream)
|
||||
and residual.residual is None
|
||||
and residual.pending is None
|
||||
)
|
||||
or post_residual_addition is not None
|
||||
)
|
||||
if capture_output is not None:
|
||||
hidden_states = (
|
||||
residual.complete(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else reduce_output(hidden_states)
|
||||
)
|
||||
if capture_before_read:
|
||||
# Embeddings precede enter; HF deepstack capture precedes the
|
||||
# extra addition. Neither is the residual returned by the read.
|
||||
value, previous = (
|
||||
residual.finish(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else (hidden_states, residual)
|
||||
)
|
||||
if previous is None:
|
||||
capture_output(value)
|
||||
else:
|
||||
capture_output(value + previous, owned=True)
|
||||
hidden_states, residual = self.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
@@ -651,6 +728,17 @@ class LayerCommunicator:
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if capture_output is not None and not capture_before_read:
|
||||
value = residual.residual
|
||||
move = self.attn.entry(forward_batch).capture_move
|
||||
if move is not None:
|
||||
value = move(value, forward_batch=forward_batch)
|
||||
keeps = self.attn.entry(forward_batch).capture_preserves_residual
|
||||
capture_output(
|
||||
value,
|
||||
owned=self.attn.entry(forward_batch).capture_move_allocates
|
||||
or (move is None and keeps is not None and keeps(value, forward_batch)),
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
residual_value = residual.residual
|
||||
move = self.attn.entry(forward_batch).input_move
|
||||
@@ -663,36 +751,18 @@ class LayerCommunicator:
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
if (
|
||||
gathered_last_layer_output is residual_value
|
||||
# An accumulator that copies on append already holds a snapshot.
|
||||
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
|
||||
and not self._post_attn_residual_is_read_only(
|
||||
residual_value, forward_batch
|
||||
)
|
||||
):
|
||||
gathered_last_layer_output = residual_value.clone()
|
||||
captured_last_layer_outputs.append(gathered_last_layer_output)
|
||||
keeps = self.attn.entry(forward_batch).capture_preserves_residual
|
||||
captured_last_layer_outputs.capture(
|
||||
gathered_last_layer_output,
|
||||
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
or (
|
||||
move is None
|
||||
and keeps is not None
|
||||
and keeps(residual_value, forward_batch)
|
||||
),
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def _post_attn_residual_is_read_only(
|
||||
self, residual: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
|
||||
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
|
||||
|
||||
Of the base fused entry's kernels only flashinfer's writes a fresh
|
||||
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
|
||||
kernel and every plain norm fold into ``residual`` in place. It runs when
|
||||
``prepare_mlp`` selected a fused entry that may return a new residual and
|
||||
the batch is not input-scattered.
|
||||
"""
|
||||
return (
|
||||
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
|
||||
def publish_attn_lora_layout(self) -> None:
|
||||
"""Attention consumes the DP-local token batch."""
|
||||
if self._publish_lora_layout:
|
||||
@@ -829,7 +899,6 @@ class LayerCommunicator:
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -100,6 +100,23 @@ def finish_layer_stack(
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
|
||||
def norm_output(hidden_states, residual, norm, capture_output=None):
|
||||
"""The final norm and an optional capture of the same updated residual.
|
||||
|
||||
Keep add and norm together: normalizing a separately rounded residual is
|
||||
not numerically equivalent to the fused kernel's FP32 accumulation. The
|
||||
capture callback owns retention of the borrowed residual storage.
|
||||
"""
|
||||
if residual is None:
|
||||
if capture_output is not None:
|
||||
capture_output(hidden_states)
|
||||
return norm(hidden_states)
|
||||
hidden_states, residual = norm(hidden_states, residual)
|
||||
if capture_output is not None:
|
||||
capture_output(residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def from_pp(
|
||||
tensors: PPProxyTensors,
|
||||
*,
|
||||
|
||||
@@ -235,8 +235,8 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
|
||||
tensor, output_buffer.tensors[key], num_tokens
|
||||
)
|
||||
return
|
||||
if isinstance(output, (list, tuple)) and isinstance(
|
||||
output_buffer, type(output)
|
||||
if (isinstance(output, list) and isinstance(output_buffer, list)) or (
|
||||
isinstance(output, tuple) and isinstance(output_buffer, tuple)
|
||||
):
|
||||
if len(output) != len(output_buffer):
|
||||
raise ValueError(
|
||||
|
||||
@@ -31,6 +31,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -722,7 +726,8 @@ class BailingMoEBlock(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
@@ -730,6 +735,7 @@ class BailingMoEBlock(nn.Module):
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -822,16 +828,9 @@ class BailingMoEModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -843,6 +842,9 @@ class BailingMoEModel(nn.Module):
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1178,10 +1179,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -1370,7 +1377,11 @@ class BailingMoELinearModel(nn.Module):
|
||||
and capture_mode.need_capture()
|
||||
)
|
||||
if capture_aux:
|
||||
dspark_aux_hidden_states: List[torch.Tensor] = []
|
||||
dspark_aux_hidden_states = AuxHiddenStateList()
|
||||
|
||||
def capture_output(value, *, owned=False):
|
||||
if value.shape[0] != 0:
|
||||
dspark_aux_hidden_states.capture(value, owned=owned)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -1381,13 +1392,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
capture_output=capture_output
|
||||
if capture_aux
|
||||
and i > self.start_layer
|
||||
and i - 1 in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if capture_aux and i in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, skip_empty=True
|
||||
)
|
||||
if snapshot is not None:
|
||||
dspark_aux_hidden_states.append(snapshot)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
@@ -1399,10 +1409,16 @@ class BailingMoELinearModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
capture_output
|
||||
if capture_aux and self.end_layer - 1 in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
if capture_aux and len(dspark_aux_hidden_states) > 0:
|
||||
return hidden_states, dspark_aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -936,13 +937,17 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1103,22 +1108,18 @@ class Glm4MoeModel(nn.Module):
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -625,12 +626,16 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
getattr(self, "_gfx95_quant_format", ""),
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=getattr(self, "_gfx95_quant_format", ""),
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -800,16 +805,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
normal_end_layer = self.first_k_dense_replace
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -817,6 +815,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_distribution import (
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -918,13 +919,17 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
zero_allocator: Optional[BumpAllocator] = None,
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
capture_output=None,
|
||||
):
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1139,7 +1144,7 @@ class Glm5NextModel(nn.Module):
|
||||
normal_end_layer = self.first_k_dense_replace
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
topk_indices = None
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
# NOTE: torch dynamo does not support graph break in context manager
|
||||
@@ -1149,22 +1154,16 @@ class Glm5NextModel(nn.Module):
|
||||
else get_global_expert_distribution_recorder().with_current_layer(i)
|
||||
)
|
||||
with ctx:
|
||||
if i in self.layers_to_capture:
|
||||
if self.dflash_capture and self.config.mhc:
|
||||
# hc_post has completed the streams; contracting them
|
||||
# creates the retained value without copying the wide input.
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
|
||||
else:
|
||||
hidden_states, aux_hidden_state = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
|
||||
def capture_output(aux_hidden_state, *, owned=False):
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(aux_hidden_state)
|
||||
owned = owned or (self.dflash_capture and self.config.mhc)
|
||||
if self.enable_a2a_moe and i > self.first_k_dense_replace:
|
||||
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
|
||||
aux_hidden_state, dim=0
|
||||
)
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
group = get_parallel().attn_tp_group
|
||||
aux_hidden_state = group.all_gather(aux_hidden_state, dim=0)
|
||||
owned = owned or group.world_size > 1
|
||||
aux_hidden_states.capture(aux_hidden_state, owned=owned)
|
||||
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
positions,
|
||||
@@ -1174,6 +1173,9 @@ class Glm5NextModel(nn.Module):
|
||||
zero_allocator,
|
||||
gemm_output_zero_allocator,
|
||||
prev_topk_indices=topk_indices,
|
||||
capture_output=capture_output
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -618,9 +619,15 @@ class GptOssDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -710,24 +717,19 @@ class GptOssModel(nn.Module):
|
||||
|
||||
# Capture hidden-state boundaries: boundary 0 is the embedding output,
|
||||
# and boundary i + 1 is the output after transformer block i.
|
||||
aux_hidden_states = []
|
||||
if self.start_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[self.start_layer].layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if i + 1 in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
@@ -741,10 +743,20 @@ class GptOssModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -644,7 +648,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
meta_mlp: nn.ModuleList,
|
||||
captured_last_layer_outputs: list[torch.Tensor] | None = None,
|
||||
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
del kwargs
|
||||
@@ -764,7 +768,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
|
||||
@@ -19,6 +19,7 @@ from torch import nn
|
||||
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -475,9 +476,15 @@ class LagunaDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -589,18 +596,17 @@ class LagunaModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
@@ -613,14 +619,16 @@ class LagunaModel(nn.Module):
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
|
||||
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -30,6 +30,10 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -997,7 +1001,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = (
|
||||
@@ -1149,7 +1153,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -35,6 +35,10 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1369,7 +1373,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
@@ -1484,7 +1488,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -40,6 +40,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1070,7 +1074,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -1183,7 +1187,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -23,7 +23,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
import triton
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
|
||||
from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
fused_qkvzba_split_reshape_cat_contiguous,
|
||||
@@ -41,6 +40,12 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1540,7 +1545,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
@@ -1825,7 +1830,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
# Pass through decoder layers
|
||||
for layer_idx in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[layer_idx]
|
||||
|
||||
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -882,7 +883,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -892,6 +894,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -15,6 +15,10 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -550,7 +554,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
@@ -826,7 +830,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
@@ -931,7 +935,7 @@ class Qwen3NextModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -102,16 +103,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(
|
||||
self.layers[self.start_layer : self.end_layer]
|
||||
):
|
||||
layer_idx += self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
|
||||
if self.use_hf_deepstack_order:
|
||||
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
|
||||
@@ -126,6 +122,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
forward_batch,
|
||||
residual,
|
||||
post_residual_addition=deepstack_embeds,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
else:
|
||||
# Inference path: add deepstack directly to hidden_states at the end of the layer
|
||||
@@ -135,6 +134,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if (
|
||||
input_deepstack_embeds is not None
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather,
|
||||
@@ -1748,7 +1749,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
if self.has_ple
|
||||
else None
|
||||
)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
if i + 1 < self.end_layer:
|
||||
|
||||
@@ -74,6 +74,7 @@ patches:
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
@@ -131,6 +132,7 @@ patches:
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -172,7 +172,6 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel():
|
||||
assert fusions[1].run == base
|
||||
# The workspace reduces over the TP group, which is the attention-TP group here.
|
||||
assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2
|
||||
assert all(f.may_return_new_residual for f in fusions)
|
||||
# A residual on each rank's slice is gathered first, which the
|
||||
# workspace does not do.
|
||||
comm._declared = SimpleNamespace(
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Aux owns retained storage, including captures from reusable gather buffers."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker
|
||||
from sglang.srt.layers.communicator import LayerCommunicator as BoundaryPlan
|
||||
from sglang.srt.layers.communicator.residual.access import norm_output
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestAuxStorage(CustomTestCase):
|
||||
def test_constructed_layer_binds_read_only_residual_capture(self):
|
||||
import test_declared_decoder_boundary as fixture
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers import layernorm
|
||||
from sglang.test.communicator_patch import patch_communicator
|
||||
|
||||
parallel = fixture.parallel_of(attn_dp=1, attn_tp=2)
|
||||
with (
|
||||
fixture.planning(parallel),
|
||||
patch_communicator("_use_aiter", False),
|
||||
patch_communicator("apply_flashinfer_allreduce_fusion", return_value=True),
|
||||
):
|
||||
layer = comm.LayerCommunicator(
|
||||
layer_facts=fixture.layer_facts(1, 3),
|
||||
input_layernorm=layernorm.RMSNorm(4),
|
||||
post_attention_layernorm=layernorm.RMSNorm(4),
|
||||
)
|
||||
keeps = layer._steps.attention.capture_preserves_residual
|
||||
self.assertIsNotNone(keeps)
|
||||
self.assertTrue(keeps(torch.ones(2, 4), SimpleNamespace()))
|
||||
|
||||
def test_breakable_graph_copies_aux_collector_into_plain_list(self):
|
||||
from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import (
|
||||
BreakableCudaGraphBackend,
|
||||
)
|
||||
|
||||
backend = BreakableCudaGraphBackend.__new__(BreakableCudaGraphBackend)
|
||||
source = AuxHiddenStateList([torch.full((4, 3), 7.0)])
|
||||
buffers = [torch.zeros(4, 3)]
|
||||
backend._copy_output_to_buffer(
|
||||
(torch.ones(4, 3), source), (torch.zeros(4, 3), buffers), 2
|
||||
)
|
||||
torch.testing.assert_close(buffers[0][:2], torch.full((2, 3), 7.0))
|
||||
self.assertEqual(buffers[0][2:].count_nonzero(), 0)
|
||||
with self.assertRaises(TypeError):
|
||||
backend._copy_output_to_buffer(source, tuple(buffers), 2)
|
||||
|
||||
def test_capture_move_owns_gather_but_not_slice(self):
|
||||
from sglang.srt.layers.communicator.boundary import Boundary
|
||||
from sglang.srt.layers.communicator.layout import Layout, TokenAxis
|
||||
from sglang.test.communicator_patch import patch_communicator
|
||||
|
||||
full = Layout(frozenset())
|
||||
sharded = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
source = torch.ones(2, 3)
|
||||
for rows, target, owns in ((sharded, full, True), (full, sharded, False)):
|
||||
edge = SimpleNamespace(
|
||||
residual_to=rows, produced=SimpleNamespace(layout=target)
|
||||
)
|
||||
boundary = Boundary(edge=edge, prepare=None)
|
||||
self.assertEqual(boundary.capture_move_allocates, owns)
|
||||
with (
|
||||
patch_communicator(
|
||||
"_redistribute_from_attn_tp_shards",
|
||||
side_effect=lambda x: torch.cat((x, x)),
|
||||
),
|
||||
patch_communicator(
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(attn_tp_size=2, attn_tp_rank=0),
|
||||
),
|
||||
):
|
||||
value = boundary.capture_move(source, forward_batch=None)
|
||||
outputs = AuxHiddenStateList()
|
||||
if owns:
|
||||
with patch.object(
|
||||
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
|
||||
):
|
||||
outputs.capture(value, owned=boundary.capture_move_allocates)
|
||||
self.assertIs(outputs[0], value)
|
||||
else:
|
||||
outputs.capture(value, owned=boundary.capture_move_allocates)
|
||||
self.assertIsNot(outputs[0], value)
|
||||
|
||||
def test_list_snapshots_borrowed_views_and_reused_buffers(self):
|
||||
outputs = AuxHiddenStateList()
|
||||
buffer = torch.full((4, 3), 2.0)
|
||||
outputs.capture(buffer[:2])
|
||||
buffer.fill_(5)
|
||||
outputs.capture(buffer[:2])
|
||||
buffer.zero_()
|
||||
torch.testing.assert_close(outputs[0], torch.full((2, 3), 2.0))
|
||||
torch.testing.assert_close(outputs[1], torch.full((2, 3), 5.0))
|
||||
|
||||
def test_list_adopts_owned_output_without_copying(self):
|
||||
outputs = AuxHiddenStateList()
|
||||
value = torch.ones(2, 3)
|
||||
with patch.object(
|
||||
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
|
||||
):
|
||||
outputs.capture(value, owned=True)
|
||||
self.assertIs(outputs[0], value)
|
||||
|
||||
def boundary(self, stream, *, move=None):
|
||||
boundary = BoundaryPlan.__new__(BoundaryPlan)
|
||||
boundary.input_layernorm = None
|
||||
boundary._context = None
|
||||
boundary.prepare_attn = Mock(return_value=(stream.residual, stream))
|
||||
boundary.attn = SimpleNamespace(
|
||||
entry=lambda _: SimpleNamespace(
|
||||
input_move=move,
|
||||
capture_move=None,
|
||||
capture_move_allocates=False,
|
||||
capture_preserves_residual=None,
|
||||
)
|
||||
)
|
||||
return boundary
|
||||
|
||||
def test_packer_receives_borrowed_boundary_value_without_intermediate_clone(self):
|
||||
for callback in (False, True):
|
||||
with self.subTest(callback=callback):
|
||||
value = torch.full((2, 3), 4.0)
|
||||
stream = ResidualStream(value)
|
||||
stage = self.boundary(stream)
|
||||
outputs = AuxHiddenStatePacker(1)
|
||||
kwargs = (
|
||||
{"capture_output": outputs.capture}
|
||||
if callback
|
||||
else {"captured_last_layer_outputs": outputs}
|
||||
)
|
||||
with patch.object(
|
||||
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
|
||||
):
|
||||
stage.prepare_attn_and_capture_last_layer_outputs(
|
||||
value, stream, None, **kwargs
|
||||
)
|
||||
value.zero_()
|
||||
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
|
||||
|
||||
def test_gather_storage_is_borrowed_even_if_it_is_a_different_tensor(self):
|
||||
source = torch.ones(2, 3)
|
||||
gathered = torch.full((4, 3), 7.0)
|
||||
stream = ResidualStream(source)
|
||||
move = Mock(return_value=gathered)
|
||||
stage = self.boundary(stream, move=move)
|
||||
outputs = AuxHiddenStateList()
|
||||
stage.prepare_attn_and_capture_last_layer_outputs(source, stream, None, outputs)
|
||||
gathered.zero_()
|
||||
source.zero_()
|
||||
torch.testing.assert_close(outputs[0], torch.full((4, 3), 7.0))
|
||||
move.assert_called_once()
|
||||
|
||||
def test_embedding_and_final_norm_capture_write_directly_to_packer(self):
|
||||
for residual in (None, torch.ones(2, 3)):
|
||||
with self.subTest(residual=residual):
|
||||
hidden = torch.full((2, 3), 4.0)
|
||||
outputs = AuxHiddenStatePacker(1)
|
||||
updated = torch.full((2, 3), 5.0)
|
||||
norm = Mock(
|
||||
return_value=hidden if residual is None else (hidden, updated)
|
||||
)
|
||||
with patch.object(
|
||||
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
|
||||
):
|
||||
norm_output(hidden, residual, norm, outputs.capture)
|
||||
hidden.zero_()
|
||||
updated.zero_()
|
||||
torch.testing.assert_close(
|
||||
outputs.finalize(),
|
||||
torch.full((2, 3), 4.0 if residual is None else 5.0),
|
||||
)
|
||||
|
||||
def test_initial_capture_precedes_in_place_prepare_without_intermediate_clone(self):
|
||||
hidden = torch.full((2, 3), 4.0)
|
||||
stream = ResidualStream()
|
||||
outputs = AuxHiddenStatePacker(1)
|
||||
stage = self.boundary(stream)
|
||||
|
||||
def prepare(value, stream, batch, **kwargs):
|
||||
value.zero_()
|
||||
stream.write(value)
|
||||
return value, stream
|
||||
|
||||
stage.prepare_attn = prepare
|
||||
with patch.object(
|
||||
torch.Tensor, "clone", side_effect=AssertionError("extra clone")
|
||||
):
|
||||
stage.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden, stream, None, capture_output=outputs.capture
|
||||
)
|
||||
torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -261,11 +261,9 @@ class TestNonAlternatingEdges(CustomTestCase):
|
||||
residual=full,
|
||||
residual_to=full,
|
||||
)
|
||||
over_tp = comm.FusedMlpInput(
|
||||
completes=SumGroup.TP, run=MagicMock(), may_return_new_residual=False
|
||||
)
|
||||
over_tp = comm.FusedMlpInput(completes=SumGroup.TP, run=MagicMock())
|
||||
over_attention_tp = comm.FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False
|
||||
completes=SumGroup.ATTN_TP, run=MagicMock()
|
||||
)
|
||||
boundary = make_boundary(edge, fusions=(over_tp, over_attention_tp))
|
||||
self.assertEqual(boundary.fused, (over_tp,))
|
||||
@@ -370,9 +368,7 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
|
||||
|
||||
def test_fused_kernels_take_only_a_plain_add(self):
|
||||
full = rows(sizes(tp=2))
|
||||
fused = comm.FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False
|
||||
)
|
||||
fused = comm.FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock())
|
||||
carried = MagicMock()
|
||||
for update, offered in ((comm.ADD, True), (_WrittenIn(), False)):
|
||||
with self.subTest(adds_plainly=offered):
|
||||
@@ -569,9 +565,7 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase):
|
||||
torch.testing.assert_close(out, expected + 100)
|
||||
|
||||
def test_a_fused_kernel_runs_only_for_a_read_that_leaves_the_residual(self):
|
||||
fused = FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=True
|
||||
)
|
||||
fused = FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock())
|
||||
for norms_plainly in (True, False):
|
||||
with self.subTest(norms_plainly=norms_plainly):
|
||||
step, tried = self._ffn_input(
|
||||
|
||||
@@ -276,9 +276,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
(entry,) if fuses else (),
|
||||
)
|
||||
self.assertIs(
|
||||
any(
|
||||
f.may_return_new_residual for f in communicator._steps.ffn.fused
|
||||
),
|
||||
bool(communicator._steps.ffn.fused),
|
||||
fuses,
|
||||
)
|
||||
self.assertFalse(communicator._steps.returns_over_dp)
|
||||
@@ -1095,11 +1093,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
|
||||
norm only where those are the steps, and only if it completes the sum the
|
||||
attention output owes."""
|
||||
|
||||
def fused(self, completes, may_return_new_residual):
|
||||
def fused(self, completes):
|
||||
return comm.FusedMlpInput(
|
||||
completes=completes,
|
||||
run=lambda h, r, fb: None,
|
||||
may_return_new_residual=may_return_new_residual,
|
||||
)
|
||||
|
||||
def select(self, *, attn_dp, fusions):
|
||||
@@ -1125,45 +1122,18 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
|
||||
return steps, fused
|
||||
|
||||
def test_a_kernel_over_another_group_is_not_chosen(self):
|
||||
over_tp = self.fused(SumGroup.TP, True)
|
||||
over_attention_tp = self.fused(SumGroup.ATTN_TP, False)
|
||||
over_tp = self.fused(SumGroup.TP)
|
||||
over_attention_tp = self.fused(SumGroup.ATTN_TP)
|
||||
steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp))
|
||||
self.assertIs(steps.func, comm_ops._mlp_input_without_dp)
|
||||
self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,))
|
||||
self.assertEqual(chosen, (over_attention_tp,))
|
||||
|
||||
def test_no_kernel_is_chosen_where_rows_are_gathered(self):
|
||||
steps, chosen = self.select(
|
||||
attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP, True),)
|
||||
)
|
||||
steps, chosen = self.select(attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP),))
|
||||
self.assertIs(steps.func, comm_ops._mlp_input_dp_partial)
|
||||
self.assertEqual(chosen, ())
|
||||
|
||||
def test_aux_capture_follows_the_chosen_kernels(self):
|
||||
for kernels, expected in (
|
||||
((self.fused(SumGroup.ATTN_TP, False),), False),
|
||||
((self.fused(SumGroup.ATTN_TP, True),), True),
|
||||
((self.fused(SumGroup.TP, True),), False),
|
||||
):
|
||||
with self.subTest(kernels=kernels):
|
||||
|
||||
class Declaring(LayerCommunicator):
|
||||
def _select_mlp_input_fusions(self):
|
||||
return kernels
|
||||
|
||||
with planning(parallel_of(attn_dp=1, attn_tp=2)):
|
||||
communicator = Declaring(
|
||||
layer_facts=layer_facts(1, 3),
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertIs(
|
||||
any(
|
||||
f.may_return_new_residual for f in communicator._steps.ffn.fused
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
"""While a LayerNorm SP region is active, the layer runs the steps its
|
||||
@@ -1492,7 +1462,6 @@ class TestOneRepresentation(CustomTestCase):
|
||||
"_ffn_output",
|
||||
"_postprocess_scatters_to_local_tokens",
|
||||
"_ffn_sum_is_movable",
|
||||
"_mlp_input_may_return_new_residual",
|
||||
)
|
||||
|
||||
def test_the_layer_holds_steps_only(self):
|
||||
|
||||
@@ -4,9 +4,11 @@ from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
BoundarySteps,
|
||||
EdgeDecl,
|
||||
LayerCommunicator,
|
||||
Layout,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
@@ -162,6 +164,107 @@ class TestResidualStream(CustomTestCase):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.stream.input(hidden.clone())
|
||||
|
||||
def test_capture_and_compute_share_the_fused_read(self):
|
||||
boundary = LayerCommunicator.__new__(LayerCommunicator)
|
||||
events = []
|
||||
outputs = AuxHiddenStateList()
|
||||
|
||||
def prepare(hidden, stream, batch, **kwargs):
|
||||
events.append("add_norm")
|
||||
self.assertIsNone(stream.pending.owed)
|
||||
# A fused read returns both the norm output and its updated residual.
|
||||
updated = hidden + stream.residual
|
||||
stream.write(updated)
|
||||
return updated * 3, stream
|
||||
|
||||
def capture(value, *, owned=False):
|
||||
events.append("capture")
|
||||
outputs.capture(value, owned=owned)
|
||||
|
||||
boundary.prepare_attn = Mock(side_effect=prepare)
|
||||
boundary.attn = SimpleNamespace(
|
||||
entry=lambda batch: SimpleNamespace(
|
||||
capture_move=None,
|
||||
capture_move_allocates=False,
|
||||
capture_preserves_residual=None,
|
||||
)
|
||||
)
|
||||
output, stream = boundary.prepare_attn_and_capture_last_layer_outputs(
|
||||
self.hidden, self.stream, None, capture_output=capture
|
||||
)
|
||||
self.assertEqual(events, ["add_norm", "capture"])
|
||||
self.group.all_reduce.assert_called_once()
|
||||
boundary.prepare_attn.assert_called_once()
|
||||
torch.testing.assert_close(output, torch.full((2, 4), 15.0))
|
||||
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
|
||||
stream.residual.zero_()
|
||||
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
|
||||
|
||||
def test_deepstack_capture_precedes_the_extra_addition(self):
|
||||
boundary = LayerCommunicator.__new__(LayerCommunicator)
|
||||
extra = torch.full_like(self.partial, 7.0)
|
||||
outputs = AuxHiddenStateList()
|
||||
|
||||
def prepare(hidden, stream, batch, **kwargs):
|
||||
self.assertEqual(len(outputs), 1)
|
||||
self.assertIs(kwargs["post_residual_addition"], extra)
|
||||
updated = hidden + stream.residual + extra
|
||||
stream.write(updated)
|
||||
return updated * 3, stream
|
||||
|
||||
boundary.prepare_attn = Mock(side_effect=prepare)
|
||||
boundary.attn = SimpleNamespace(
|
||||
entry=lambda batch: SimpleNamespace(
|
||||
capture_move=None,
|
||||
capture_move_allocates=False,
|
||||
capture_preserves_residual=None,
|
||||
)
|
||||
)
|
||||
output, _ = boundary.prepare_attn_and_capture_last_layer_outputs(
|
||||
self.hidden,
|
||||
self.stream,
|
||||
None,
|
||||
post_residual_addition=extra,
|
||||
capture_output=outputs.capture,
|
||||
)
|
||||
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
|
||||
torch.testing.assert_close(output, torch.full((2, 4), 36.0))
|
||||
self.group.all_reduce.assert_called_once()
|
||||
|
||||
def test_capture_restores_the_producers_rows_after_a_scattered_read(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.layers.communicator import TokenAxis
|
||||
|
||||
full = Layout(frozenset())
|
||||
local = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
boundary = make_boundary(
|
||||
EdgeDecl(StageOutput(full), StageInput(local), full, local)
|
||||
)
|
||||
shard = torch.full((2, 4), 3.0)
|
||||
expected = torch.cat([shard, torch.full_like(shard, 7.0)])
|
||||
with patch(
|
||||
"sglang.srt.layers.communicator.ops._redistribute_from_attn_tp_shards",
|
||||
return_value=expected,
|
||||
) as gather:
|
||||
captured = boundary.capture_move(shard, forward_batch=None)
|
||||
gather.assert_called_once_with(shard)
|
||||
torch.testing.assert_close(captured, expected)
|
||||
torch.testing.assert_close(shard, torch.full((2, 4), 3.0))
|
||||
|
||||
def test_final_capture_uses_the_norms_residual_result(self):
|
||||
from sglang.srt.layers.communicator.residual.access import norm_output
|
||||
|
||||
outputs = AuxHiddenStateList()
|
||||
updated = torch.full_like(self.partial, 5.0)
|
||||
normalized = torch.full_like(self.partial, 13.0)
|
||||
norm = Mock(return_value=(normalized, updated))
|
||||
result = norm_output(self.partial, self.residual, norm, outputs.capture)
|
||||
self.assertIs(result, normalized)
|
||||
norm.assert_called_once_with(self.partial, self.residual)
|
||||
updated.zero_()
|
||||
torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0))
|
||||
|
||||
def test_deepstack_adds_once_after_completing_the_sum(self):
|
||||
hidden, stream = add_to_output(
|
||||
self.hidden, self.stream, torch.full((2, 4), 7.0)
|
||||
|
||||
@@ -118,6 +118,9 @@ class DeferringLayer(nn.Module):
|
||||
# later norms mutate the residual; captured snapshots must stay intact
|
||||
residual.add_(hidden_states)
|
||||
stream.write(residual)
|
||||
capture = kwargs.get("capture_output")
|
||||
if capture is not None:
|
||||
capture(residual.clone())
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
partial = torch.full_like(hidden_states, 0.5)
|
||||
hidden_states, residual = ffn_exit.finish(partial, stream)
|
||||
@@ -130,7 +133,7 @@ class SumNorm(nn.Module):
|
||||
def forward(self, hidden_states, residual=None, **kwargs):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states + residual, residual
|
||||
return hidden_states + residual, hidden_states + residual
|
||||
|
||||
|
||||
def build_model(model_cls, *, defer, capture):
|
||||
@@ -211,6 +214,29 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase):
|
||||
reduce.call_count, NUM_LAYERS - 1 if defer else 0
|
||||
)
|
||||
|
||||
def test_terminal_capture_uses_the_final_norm_boundary(self):
|
||||
inputs = torch.ones(2, 4)
|
||||
batch = SimpleNamespace(
|
||||
can_run_tbo=False,
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
capture_hidden_mode=SimpleNamespace(need_capture=lambda: True),
|
||||
)
|
||||
for cls in (GptOssModel, LagunaModel, BailingMoELinearModel):
|
||||
with self.subTest(model=cls.__name__):
|
||||
model = build_model(cls, defer=True, capture=True)
|
||||
model.layers_to_capture = [
|
||||
NUM_LAYERS - 1 if cls is BailingMoELinearModel else NUM_LAYERS
|
||||
]
|
||||
result, snapshots = model(
|
||||
input_ids=None,
|
||||
positions=None,
|
||||
forward_batch=batch,
|
||||
input_embeds=inputs.clone(),
|
||||
)
|
||||
self.assertEqual(len(snapshots), 1)
|
||||
torch.testing.assert_close(snapshots[0], inputs + NUM_LAYERS)
|
||||
torch.testing.assert_close(result, inputs + NUM_LAYERS)
|
||||
|
||||
|
||||
class TestPipelineResidualReception(CustomTestCase):
|
||||
def test_models_keep_the_received_residual_contribution(self):
|
||||
@@ -238,6 +264,29 @@ class TestPipelineResidualReception(CustomTestCase):
|
||||
)
|
||||
torch.testing.assert_close(result, initial_residual + NUM_LAYERS)
|
||||
|
||||
def test_bailing_capture_does_not_include_a_previous_pipeline_ranks_layer(self):
|
||||
model = build_model(BailingMoELinearModel, defer=True, capture=True)
|
||||
model.pp_group.is_first_rank = False
|
||||
model.start_layer = 2
|
||||
model.layers_to_capture = [model.start_layer - 1]
|
||||
inputs = torch.zeros(2, 4)
|
||||
residual = torch.full_like(inputs, 0.25)
|
||||
batch = SimpleNamespace(
|
||||
can_run_tbo=False,
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
capture_hidden_mode=SimpleNamespace(need_capture=lambda: True),
|
||||
)
|
||||
result = model(
|
||||
None,
|
||||
None,
|
||||
batch,
|
||||
pp_proxy_tensors=PPProxyTensors(
|
||||
{"hidden_states": inputs, "residual": residual}
|
||||
),
|
||||
)
|
||||
self.assertIsInstance(result, torch.Tensor)
|
||||
torch.testing.assert_close(result, torch.full_like(inputs, 2.25))
|
||||
|
||||
def test_written_streams_do_not_read_a_separate_residual(self):
|
||||
comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
comm_instance._residual = SimpleNamespace(
|
||||
|
||||
Reference in New Issue
Block a user