[Refactor] Capture auxiliary states at residual reads (#41550)
This commit is contained in:
@@ -9,6 +9,18 @@ import torch
|
||||
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
|
||||
|
||||
|
||||
class AuxHiddenStateList(list):
|
||||
"""Retain aux snapshots independently of storage reused by later layers."""
|
||||
|
||||
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
|
||||
"""Copy borrowed storage, or adopt a value whose ownership is transferred.
|
||||
|
||||
Views and reusable communication buffers are borrowed even when they
|
||||
are different tensor objects from the source.
|
||||
"""
|
||||
self.append(hidden if owned else hidden.clone())
|
||||
|
||||
|
||||
class AuxHiddenStatePacker:
|
||||
"""Drop-in for the ``[]`` a model collects Eagle3/DFlash captures into.
|
||||
|
||||
@@ -17,9 +29,6 @@ class AuxHiddenStatePacker:
|
||||
Assumes all captures share leading shape and feature size.
|
||||
"""
|
||||
|
||||
# ``append`` copies, so producers need not clone a tensor they later mutate.
|
||||
copies_on_append = True
|
||||
|
||||
def __init__(self, num_captures: int) -> None:
|
||||
self._num_captures = int(num_captures)
|
||||
self._buffer: Optional[torch.Tensor] = None
|
||||
@@ -37,6 +46,10 @@ class AuxHiddenStatePacker:
|
||||
self._buffer[..., start : start + self._feature_size].copy_(hidden)
|
||||
self._idx += 1
|
||||
|
||||
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
|
||||
"""Write directly to the final packed buffer, without an intermediate copy."""
|
||||
self.append(hidden)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._idx
|
||||
|
||||
@@ -49,8 +62,8 @@ class AuxHiddenStatePacker:
|
||||
return self._buffer
|
||||
|
||||
|
||||
# What a model hands down the capture path: a plain list, or a packer writing in place.
|
||||
AuxHiddenStateAccumulator = Union[List[torch.Tensor], AuxHiddenStatePacker]
|
||||
# Both collectors accept borrowed values through capture; storage belongs to aux.
|
||||
AuxHiddenStateAccumulator = Union[AuxHiddenStateList, AuxHiddenStatePacker]
|
||||
|
||||
|
||||
def pack_aux_hidden_states(aux_hidden_states: AuxHiddenStates) -> torch.Tensor:
|
||||
|
||||
@@ -47,6 +47,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
_mlp_input_slice,
|
||||
_mlp_input_without_dp,
|
||||
_read_input,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
@@ -479,8 +480,6 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
# It may hand back a new residual and leave the one it took unchanged.
|
||||
may_return_new_residual: bool
|
||||
|
||||
|
||||
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
@@ -548,6 +547,10 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
handoff: Optional[Callable] = None
|
||||
# The fused kernels prepare tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Return the updated residual to the producer's rows for aux capture.
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
@@ -624,6 +627,19 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
|
||||
)
|
||||
|
||||
@property
|
||||
def capture_move_allocates(self) -> bool:
|
||||
"""Returning through a gather allocates; returning through a cut aliases."""
|
||||
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
|
||||
|
||||
@property
|
||||
def capture_move(self) -> Optional[Callable]:
|
||||
if self.edge.residual_to == self.edge.produced.layout:
|
||||
return None
|
||||
return partial(
|
||||
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
|
||||
)
|
||||
|
||||
|
||||
def make_boundary(
|
||||
edge: EdgeDecl,
|
||||
@@ -808,6 +824,8 @@ def _select_boundary_steps(
|
||||
input_rows=into_attention.input_rows,
|
||||
input_move=into_attention.input_move,
|
||||
handoff=attention_handoff,
|
||||
capture_move=into_attention.capture_move,
|
||||
capture_move_allocates=into_attention.capture_move_allocates,
|
||||
),
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
|
||||
@@ -250,7 +250,6 @@ class CuteDSLFusion:
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
layer,
|
||||
),
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
return ()
|
||||
|
||||
@@ -59,9 +59,11 @@ from sglang.srt.layers.communicator.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
_all_reduce_then_to_local_tokens,
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
_mlp_input_without_dp,
|
||||
_redistribute_output,
|
||||
_reduce_and_redistribute_output_step,
|
||||
_to_local_tokens,
|
||||
@@ -79,6 +81,7 @@ from sglang.srt.layers.communicator.residual.access import (
|
||||
finish_layer_stack,
|
||||
fold,
|
||||
from_pp,
|
||||
norm_output,
|
||||
snapshot,
|
||||
written,
|
||||
)
|
||||
@@ -94,6 +97,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
@@ -244,6 +248,7 @@ class LayerCommunicator:
|
||||
add_to_output = staticmethod(add_to_output)
|
||||
buffer = staticmethod(buffer)
|
||||
fold = staticmethod(fold)
|
||||
norm_output = staticmethod(norm_output)
|
||||
written = staticmethod(written)
|
||||
finish_layer_stack = staticmethod(finish_layer_stack)
|
||||
|
||||
@@ -361,6 +366,46 @@ class LayerCommunicator:
|
||||
else None
|
||||
)
|
||||
|
||||
# Bind aux ownership to the selected input implementation once.
|
||||
for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"):
|
||||
steps = self.__dict__[name]
|
||||
if steps is None or steps.ffn is None:
|
||||
continue
|
||||
prepare = steps.ffn.prepare
|
||||
if isinstance(prepare, partial):
|
||||
prepare = prepare.keywords.get("step", prepare)
|
||||
norm = self.post_attention_layernorm
|
||||
known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type(
|
||||
norm
|
||||
).forward_with_allreduce_fusion in (
|
||||
RMSNorm.forward_with_allreduce_fusion,
|
||||
GemmaRMSNorm.forward_with_allreduce_fusion,
|
||||
)
|
||||
if (
|
||||
known_norm
|
||||
and isinstance(prepare, partial)
|
||||
and prepare.func is _mlp_input_without_dp
|
||||
and not prepare.keywords["gathers_residual"]
|
||||
and steps.ffn.fused
|
||||
and steps.ffn.fused[0].run
|
||||
== self._mlp_input_reduce_output_and_update_and_read_residual
|
||||
):
|
||||
self.__dict__[name] = msgspec.structs.replace(
|
||||
steps,
|
||||
attention=msgspec.structs.replace(
|
||||
steps.attention,
|
||||
capture_preserves_residual=self._capture_keeps_residual,
|
||||
),
|
||||
)
|
||||
|
||||
def _capture_keeps_residual(self, value, forward_batch):
|
||||
# The CUDA wrapper allocates a fresh residual even on backend fallback.
|
||||
return (
|
||||
not _use_aiter
|
||||
and get_parallel().attn_tp_size > 1
|
||||
and apply_flashinfer_allreduce_fusion(value.shape[0])
|
||||
)
|
||||
|
||||
def _init_stage(self, stage: "LayerStage") -> None:
|
||||
"""A layer that is one stage: every batch runs the two boundaries its
|
||||
declarations give."""
|
||||
@@ -387,6 +432,7 @@ class LayerCommunicator:
|
||||
input_move=into.input_move,
|
||||
handoff=None if is_ffn else _hand_qkv_hook_its_input,
|
||||
fused=into.fused,
|
||||
capture_move=into.capture_move,
|
||||
)
|
||||
self._steps = BoundarySteps(
|
||||
attention=None if is_ffn else entry,
|
||||
@@ -643,7 +689,38 @@ class LayerCommunicator:
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
quant_format: str = "",
|
||||
capture_output: Optional[Callable] = None,
|
||||
):
|
||||
# Aux consumers need a materialized output before the input norm.
|
||||
# Complete communication here, preserving the existing add+norm kernel
|
||||
# and its FP32 accumulation. Its residual result also supplies capture.
|
||||
capture_before_read = capture_output is not None and (
|
||||
residual is None
|
||||
or (
|
||||
isinstance(residual, ResidualStream)
|
||||
and residual.residual is None
|
||||
and residual.pending is None
|
||||
)
|
||||
or post_residual_addition is not None
|
||||
)
|
||||
if capture_output is not None:
|
||||
hidden_states = (
|
||||
residual.complete(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else reduce_output(hidden_states)
|
||||
)
|
||||
if capture_before_read:
|
||||
# Embeddings precede enter; HF deepstack capture precedes the
|
||||
# extra addition. Neither is the residual returned by the read.
|
||||
value, previous = (
|
||||
residual.finish(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else (hidden_states, residual)
|
||||
)
|
||||
if previous is None:
|
||||
capture_output(value)
|
||||
else:
|
||||
capture_output(value + previous, owned=True)
|
||||
hidden_states, residual = self.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
@@ -651,6 +728,17 @@ class LayerCommunicator:
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if capture_output is not None and not capture_before_read:
|
||||
value = residual.residual
|
||||
move = self.attn.entry(forward_batch).capture_move
|
||||
if move is not None:
|
||||
value = move(value, forward_batch=forward_batch)
|
||||
keeps = self.attn.entry(forward_batch).capture_preserves_residual
|
||||
capture_output(
|
||||
value,
|
||||
owned=self.attn.entry(forward_batch).capture_move_allocates
|
||||
or (move is None and keeps is not None and keeps(value, forward_batch)),
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
residual_value = residual.residual
|
||||
move = self.attn.entry(forward_batch).input_move
|
||||
@@ -663,36 +751,18 @@ class LayerCommunicator:
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
if (
|
||||
gathered_last_layer_output is residual_value
|
||||
# An accumulator that copies on append already holds a snapshot.
|
||||
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
|
||||
and not self._post_attn_residual_is_read_only(
|
||||
residual_value, forward_batch
|
||||
)
|
||||
):
|
||||
gathered_last_layer_output = residual_value.clone()
|
||||
captured_last_layer_outputs.append(gathered_last_layer_output)
|
||||
keeps = self.attn.entry(forward_batch).capture_preserves_residual
|
||||
captured_last_layer_outputs.capture(
|
||||
gathered_last_layer_output,
|
||||
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
or (
|
||||
move is None
|
||||
and keeps is not None
|
||||
and keeps(residual_value, forward_batch)
|
||||
),
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def _post_attn_residual_is_read_only(
|
||||
self, residual: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
|
||||
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
|
||||
|
||||
Of the base fused entry's kernels only flashinfer's writes a fresh
|
||||
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
|
||||
kernel and every plain norm fold into ``residual`` in place. It runs when
|
||||
``prepare_mlp`` selected a fused entry that may return a new residual and
|
||||
the batch is not input-scattered.
|
||||
"""
|
||||
return (
|
||||
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
|
||||
def publish_attn_lora_layout(self) -> None:
|
||||
"""Attention consumes the DP-local token batch."""
|
||||
if self._publish_lora_layout:
|
||||
@@ -829,7 +899,6 @@ class LayerCommunicator:
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=self._mlp_input_reduce_output_and_update_and_read_residual,
|
||||
may_return_new_residual=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -100,6 +100,23 @@ def finish_layer_stack(
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
|
||||
def norm_output(hidden_states, residual, norm, capture_output=None):
|
||||
"""The final norm and an optional capture of the same updated residual.
|
||||
|
||||
Keep add and norm together: normalizing a separately rounded residual is
|
||||
not numerically equivalent to the fused kernel's FP32 accumulation. The
|
||||
capture callback owns retention of the borrowed residual storage.
|
||||
"""
|
||||
if residual is None:
|
||||
if capture_output is not None:
|
||||
capture_output(hidden_states)
|
||||
return norm(hidden_states)
|
||||
hidden_states, residual = norm(hidden_states, residual)
|
||||
if capture_output is not None:
|
||||
capture_output(residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def from_pp(
|
||||
tensors: PPProxyTensors,
|
||||
*,
|
||||
|
||||
@@ -235,8 +235,8 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
|
||||
tensor, output_buffer.tensors[key], num_tokens
|
||||
)
|
||||
return
|
||||
if isinstance(output, (list, tuple)) and isinstance(
|
||||
output_buffer, type(output)
|
||||
if (isinstance(output, list) and isinstance(output_buffer, list)) or (
|
||||
isinstance(output, tuple) and isinstance(output_buffer, tuple)
|
||||
):
|
||||
if len(output) != len(output_buffer):
|
||||
raise ValueError(
|
||||
|
||||
@@ -31,6 +31,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -722,7 +726,8 @@ class BailingMoEBlock(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
@@ -730,6 +735,7 @@ class BailingMoEBlock(nn.Module):
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -822,16 +828,9 @@ class BailingMoEModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -843,6 +842,9 @@ class BailingMoEModel(nn.Module):
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1178,10 +1179,16 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -1370,7 +1377,11 @@ class BailingMoELinearModel(nn.Module):
|
||||
and capture_mode.need_capture()
|
||||
)
|
||||
if capture_aux:
|
||||
dspark_aux_hidden_states: List[torch.Tensor] = []
|
||||
dspark_aux_hidden_states = AuxHiddenStateList()
|
||||
|
||||
def capture_output(value, *, owned=False):
|
||||
if value.shape[0] != 0:
|
||||
dspark_aux_hidden_states.capture(value, owned=owned)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -1381,13 +1392,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
capture_output=capture_output
|
||||
if capture_aux
|
||||
and i > self.start_layer
|
||||
and i - 1 in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if capture_aux and i in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, skip_empty=True
|
||||
)
|
||||
if snapshot is not None:
|
||||
dspark_aux_hidden_states.append(snapshot)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
@@ -1399,10 +1409,16 @@ class BailingMoELinearModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
capture_output
|
||||
if capture_aux and self.end_layer - 1 in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
if capture_aux and len(dspark_aux_hidden_states) > 0:
|
||||
return hidden_states, dspark_aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -936,13 +937,17 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=self.attn_quant_format,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1103,22 +1108,18 @@ class Glm4MoeModel(nn.Module):
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -625,12 +626,16 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
getattr(self, "_gfx95_quant_format", ""),
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format=getattr(self, "_gfx95_quant_format", ""),
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -800,16 +805,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
normal_end_layer = self.first_k_dense_replace
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -817,6 +815,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
forward_batch,
|
||||
residual,
|
||||
zero_allocator,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_distribution import (
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -918,13 +919,17 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
zero_allocator: Optional[BumpAllocator] = None,
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
capture_output=None,
|
||||
):
|
||||
hidden_states_orig = self.layer_communicator.buffer(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
@@ -1139,7 +1144,7 @@ class Glm5NextModel(nn.Module):
|
||||
normal_end_layer = self.first_k_dense_replace
|
||||
elif self.first_k_dense_replace < normal_start_layer:
|
||||
normal_end_layer = normal_start_layer = 0
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
topk_indices = None
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
# NOTE: torch dynamo does not support graph break in context manager
|
||||
@@ -1149,22 +1154,16 @@ class Glm5NextModel(nn.Module):
|
||||
else get_global_expert_distribution_recorder().with_current_layer(i)
|
||||
)
|
||||
with ctx:
|
||||
if i in self.layers_to_capture:
|
||||
if self.dflash_capture and self.config.mhc:
|
||||
# hc_post has completed the streams; contracting them
|
||||
# creates the retained value without copying the wide input.
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(hidden_states)
|
||||
else:
|
||||
hidden_states, aux_hidden_state = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
|
||||
def capture_output(aux_hidden_state, *, owned=False):
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(aux_hidden_state)
|
||||
owned = owned or (self.dflash_capture and self.config.mhc)
|
||||
if self.enable_a2a_moe and i > self.first_k_dense_replace:
|
||||
aux_hidden_state = get_parallel().attn_tp_group.all_gather(
|
||||
aux_hidden_state, dim=0
|
||||
)
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
group = get_parallel().attn_tp_group
|
||||
aux_hidden_state = group.all_gather(aux_hidden_state, dim=0)
|
||||
owned = owned or group.world_size > 1
|
||||
aux_hidden_states.capture(aux_hidden_state, owned=owned)
|
||||
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, topk_indices = layer(
|
||||
positions,
|
||||
@@ -1174,6 +1173,9 @@ class Glm5NextModel(nn.Module):
|
||||
zero_allocator,
|
||||
gemm_output_zero_allocator,
|
||||
prev_topk_indices=topk_indices,
|
||||
capture_output=capture_output
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if normal_end_layer != self.end_layer:
|
||||
|
||||
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -618,9 +619,15 @@ class GptOssDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -710,24 +717,19 @@ class GptOssModel(nn.Module):
|
||||
|
||||
# Capture hidden-state boundaries: boundary 0 is the embedding output,
|
||||
# and boundary i + 1 is the output after transformer block i.
|
||||
aux_hidden_states = []
|
||||
if self.start_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[self.start_layer].layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if i + 1 in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
@@ -741,10 +743,20 @@ class GptOssModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -644,7 +648,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
meta_mlp: nn.ModuleList,
|
||||
captured_last_layer_outputs: list[torch.Tensor] | None = None,
|
||||
captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
del kwargs
|
||||
@@ -764,7 +768,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
|
||||
@@ -19,6 +19,7 @@ from torch import nn
|
||||
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -475,9 +476,15 @@ class LagunaDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -589,18 +596,17 @@ class LagunaModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
@@ -613,14 +619,16 @@ class LagunaModel(nn.Module):
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
|
||||
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -30,6 +30,10 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -997,7 +1001,7 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = (
|
||||
@@ -1149,7 +1153,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -35,6 +35,10 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1369,7 +1373,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
@@ -1484,7 +1488,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -40,6 +40,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1070,7 +1074,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -1183,7 +1187,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
if forward_batch.can_run_tbo:
|
||||
hidden_states, residual = model_forward_maybe_tbo(
|
||||
layers=self.layers,
|
||||
|
||||
@@ -23,7 +23,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
import triton
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
|
||||
from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
fused_qkvzba_split_reshape_cat_contiguous,
|
||||
@@ -41,6 +40,12 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
@@ -1540,7 +1545,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
@@ -1825,7 +1830,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
# Pass through decoder layers
|
||||
for layer_idx in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[layer_idx]
|
||||
|
||||
@@ -29,6 +29,7 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -882,7 +883,8 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[List[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -892,6 +894,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -15,6 +15,10 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -550,7 +554,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs,
|
||||
):
|
||||
forward_batch = kwargs.get("forward_batch", None)
|
||||
@@ -826,7 +830,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[list[torch.Tensor]] = None,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states, residual = (
|
||||
@@ -931,7 +935,7 @@ class Qwen3NextModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -102,16 +103,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(
|
||||
self.layers[self.start_layer : self.end_layer]
|
||||
):
|
||||
layer_idx += self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
hidden_states, snapshot = layer.layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
|
||||
if self.use_hf_deepstack_order:
|
||||
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
|
||||
@@ -126,6 +122,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
forward_batch,
|
||||
residual,
|
||||
post_residual_addition=deepstack_embeds,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
else:
|
||||
# Inference path: add deepstack directly to hidden_states at the end of the layer
|
||||
@@ -135,6 +134,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if (
|
||||
input_deepstack_embeds is not None
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather,
|
||||
@@ -1748,7 +1749,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
if self.has_ple
|
||||
else None
|
||||
)
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
if i + 1 < self.end_layer:
|
||||
|
||||
Reference in New Issue
Block a user