[Refactor] Carry residual state across stage boundaries (#41549)

This commit is contained in:
Cheng Wan
2026-09-28 20:15:03 -07:00
committed by GitHub
parent 6f72cc3b64
commit 42f58b4da7
73 changed files with 1599 additions and 239 deletions
@@ -17,9 +17,10 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import (
CommunicateContext,
Layout,
reduce_output,
tbo_split_moves,
)
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
@@ -964,12 +965,25 @@ def _model_forward_tbo(
operations_strategy: OperationsStrategy,
layer_input_rows: Layout,
):
inputs["hidden_states"] = reduce_output(inputs["hidden_states"])
parent_batch = inputs["forward_batch"]
stream = inputs["residual"]
pending = stream.pending if isinstance(stream, ResidualStream) else None
inputs["hidden_states"], inputs["residual"] = finish_layer_stack(
inputs["hidden_states"], inputs["residual"], inputs["forward_batch"]
)
inputs_arr = _model_forward_tbo_split_inputs(
**inputs, layer_input_rows=layer_input_rows
)
original_hidden_states_len = inputs["hidden_states"].shape[0]
del inputs
if isinstance(stream, ResidualStream):
for part in inputs_arr:
part["hidden_states"], part["residual"] = ResidualStream.arrive(
part["hidden_states"],
part["residual"],
pending.update if pending is not None else None,
)
part["forward_batch"].residual_stream = part["residual"]
del inputs, stream, pending
context = (
empty_context()
@@ -986,7 +1000,13 @@ def _model_forward_tbo(
delta_stages=[0, operations_strategy.tbo_delta_stages],
)
return _model_forward_tbo_merge_outputs(*outputs_arr, original_hidden_states_len)
hidden_states, residual = _model_forward_tbo_merge_outputs(
*outputs_arr, original_hidden_states_len
)
parent_batch.residual_stream = (
residual if isinstance(residual, ResidualStream) else None
)
return hidden_states, residual
def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy):
@@ -1100,6 +1120,21 @@ def _model_forward_filter_inputs(
def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
stream_a, stream_b = output_a["residual"], output_b["residual"]
has_stream = isinstance(stream_a, ResidualStream)
assert has_stream == isinstance(stream_b, ResidualStream)
update = None
if has_stream:
pending_a, pending_b = stream_a.pending, stream_b.pending
assert (pending_a is None) == (pending_b is None)
if pending_a is not None:
assert pending_a.update is pending_b.update
update = pending_a.update
for output in (output_a, output_b):
output["hidden_states"], output["residual"] = finish_layer_stack(
output["hidden_states"], output["residual"], output["forward_batch"]
)
def _handle_key(name):
value_a = output_a[name]
value_b = output_b[name]
@@ -1117,7 +1152,12 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
res[slice(s1, t1)] = value_b[: t1 - s1]
return res
return _handle_key("hidden_states"), _handle_key("residual")
hidden, residual = _handle_key("hidden_states"), _handle_key("residual")
return (
ResidualStream.arrive(hidden, residual, update)
if has_stream
else (hidden, residual)
)
# -------------------------------- Utilities and wrappers ---------------------------------------
+105 -23
View File
@@ -35,6 +35,7 @@ from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
_consumer_step,
_dispatch_consumer,
_hand_qkv_hook_its_input,
_mlp_input_dp_partial,
_mlp_input_dp_replicate,
@@ -96,8 +97,9 @@ class StageOutput(msgspec.Struct, frozen=True):
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
# How the producer's output is written into the residual.
update: StageUpdate = ADD
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[StageUpdate] = ADD
class StageDecl(msgspec.Struct, frozen=True):
@@ -140,6 +142,8 @@ class EdgeDecl(msgspec.Struct, frozen=True):
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
update_capabilities: Tuple[bool, ...] = ()
class DecoderLayerEdges(msgspec.Struct, frozen=True):
@@ -159,13 +163,13 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
construction (``input_owes``); a sum the previous layer leaves for a batch
comes with the value."""
owes = sides.input_owes is not None
# The previous layer's FFN output, written in as this layer's FFN writes its
# own.
# Cross-layer arrival describes rows and owed work. The actual update
# arrives in the stream; this model family supplies only its capability.
arrived = StageOutput(
sides.input_rows,
group=sides.input_owes,
always_leaves=owes,
update=sides.ffn.output.update,
update=None,
)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
@@ -180,6 +184,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
need=sides.attention.input,
residual=sides.input_rows,
residual_to=attention_rows,
update_capabilities=(sides.ffn.output.update.adds_plainly,),
),
into_ffn=EdgeDecl(
produced=sides.attention.output,
@@ -227,22 +232,28 @@ def stage_edges(
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
update = previous.update if previous is not None else ADD
adds_plainly = previous.update.adds_plainly if previous is not None else True
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
update=update,
update=None,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows, update=update)
else StageOutput(rows, update=None)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
EdgeDecl(
produced=arrived,
need=stage.input,
residual=rows,
residual_to=during,
update_capabilities=(adds_plainly,),
),
EdgeDecl(
produced=stage.output,
need=StageInput(rows),
@@ -622,6 +633,56 @@ def make_boundary(
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> Boundary:
"""Bind only the update capabilities that can reach this edge.
Execution receives the producer's actual update from the residual stream.
A single capability needs no dispatcher; multiple capabilities select one
of the already constructed paths at the consumer.
"""
capabilities = edge.update_capabilities
if not capabilities:
if edge.produced.update is None:
raise ValueError("an arrival must declare its update capabilities")
capabilities = (edge.produced.update.adds_plainly,)
paths = {
capability: _bind_consumer(
edge,
adds_plainly=capability,
fusions=fusions,
carried_fusions=carried_fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
for capability in capabilities
}
first = next(iter(paths.values()))
if len(paths) == 1:
return first
if any(path.input_move != first.input_move for path in paths.values()):
raise NotImplementedError("update capabilities require different input moves")
return msgspec.structs.replace(
first,
prepare=partial(
_dispatch_consumer,
paths={capability: path.prepare for capability, path in paths.items()},
),
fused=tuple(
f for f in fusions if any(f in path.fused for path in paths.values())
),
)
def _bind_consumer(
edge: EdgeDecl,
*,
adds_plainly: bool,
fusions: Tuple["FusedMlpInput", ...] = (),
carried_fusions: Tuple[Callable, ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> Boundary:
"""The consumer's half of ``edge``, chosen from the edge's declarations and
the producer's update: what a value carries for a batch (a sum or a handoff
@@ -633,28 +694,45 @@ def make_boundary(
over attention CP; ``enters_stack`` for the edge into the layer stack's
first stage. The steps read only this edge's declarations, never what the
producer chose for a batch."""
update = edge.produced.update
# A fused kernel runs the add and the norm itself.
plain = update.adds_plainly and edge.need.read.norms_plainly
plain = adds_plainly and edge.need.read.norms_plainly
step, fused, input_move = _select_input_steps(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
update=update,
adds_plainly=adds_plainly,
fusions=fusions if plain else (),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
# A written stream entering the first physical layer must not run enter
# again (e.g. MHC expansion). Both alternatives are bound at construction.
written_step = None
if enters_stack:
written_step, _, _ = _select_input_steps(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
adds_plainly=adds_plainly,
fusions=fusions if plain else (),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=False,
)
return Boundary(
edge,
prepare=partial(
_consumer_step,
step=step,
adds_plainly=adds_plainly,
carried_fusions=carried_fusions if plain else (),
owes_by_construction=edge.produced.always_leaves,
written_step=written_step,
),
input_move=input_move,
fused=fused,
@@ -668,6 +746,16 @@ def make_output_boundary(
rows the layer hands on (``edge.need``), whose consumer runs in the next
layer: the postprocess that moves the output there. ``cp_moves`` for an
edge that returns across attention CP."""
update = edge.produced.update
if not getattr(update, "at_producer", False):
if not getattr(update, "can_defer_across_layers", False):
raise NotImplementedError(
"a deferred update must guarantee its lifetime across layers"
)
if not update.adds_plainly and get_parallel().pp_size > 1:
raise NotImplementedError(
"pipeline boundaries require a plain add or a producer-written residual"
)
if edge.need.layout != edge.residual_to:
raise NotImplementedError(f"{edge=}")
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
@@ -740,7 +828,7 @@ def _select_input_steps(
residual: Layout,
residual_to: Layout,
need: StageInput,
update: StageUpdate,
adds_plainly: bool,
fusions: Tuple[FusedMlpInput, ...],
force_layernorm_before_gather: bool,
residual_joins_sum: bool,
@@ -781,7 +869,6 @@ def _select_input_steps(
_mlp_input_scatter if owes is SumGroup.ATTN_TP else _mlp_input_slice,
scatters_residual=residual != residual_to,
read=read,
update=update,
),
(),
None,
@@ -793,7 +880,7 @@ def _select_input_steps(
if gathered or owes is not SumGroup.ATTN_TP:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(_mlp_input_on_residual_shard, read=read, update=update),
partial(_mlp_input_on_residual_shard, read=read),
(),
None,
)
@@ -812,7 +899,6 @@ def _select_input_steps(
layer_input=tp_reduce_scatter,
enters_stack=enters_stack,
read=read,
update=update,
),
(),
None,
@@ -826,7 +912,7 @@ def _select_input_steps(
residual=residual,
residual_to=residual_to,
need=StageInput(produced.layout, read=read),
update=update,
adds_plainly=adds_plainly,
fusions=fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=residual_joins_sum,
@@ -849,7 +935,6 @@ def _select_input_steps(
layer_input=None,
enters_stack=enters_stack,
read=read,
update=update,
),
(),
CommunicateSimpleFn._scattered_to_tp_attn_full,
@@ -878,7 +963,6 @@ def _select_input_steps(
layer_input=None,
enters_stack=enters_stack,
read=read,
update=update,
),
(),
None,
@@ -886,7 +970,7 @@ def _select_input_steps(
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
if owes is not SumGroup.ATTN_TP or not update.adds_plainly:
if owes is not SumGroup.ATTN_TP or not adds_plainly:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_residual_into_sum, read=read), (), None
# A sum over TP completes only on rows every TP rank holds: the TP group
@@ -903,7 +987,6 @@ def _select_input_steps(
fusions=tuple(f.run for f in fused),
group=owes,
read=read,
update=update,
),
fused,
None,
@@ -920,7 +1003,7 @@ def _select_input_steps(
if (
owes_attention_tp
and not force_layernorm_before_gather
and update.adds_plainly
and adds_plainly
and read.norms_plainly
):
return (
@@ -940,7 +1023,6 @@ def _select_input_steps(
reduces_attention_tp=owes_attention_tp,
places_cp_shards=places_cp_shards,
read=read,
update=update,
),
(),
None,
+103 -24
View File
@@ -72,6 +72,7 @@ from sglang.srt.layers.communicator.output import (
reduce_output,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.communicator.residual.access import (
add_to_output,
buffer,
@@ -79,6 +80,7 @@ from sglang.srt.layers.communicator.residual.access import (
fold,
from_pp,
snapshot,
written,
)
from sglang.srt.layers.communicator.residual.add_norm import (
PLAIN_RESIDUAL,
@@ -87,6 +89,7 @@ from sglang.srt.layers.communicator.residual.add_norm import (
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.communicator.residual.mhc import MHCState
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
@@ -195,6 +198,15 @@ class StageCommunicator:
``quant_format``)."""
entry = self.entry(forward_batch, steps)
context = self._layer._context
stream = residual_batch.current(forward_batch)
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
if stream is not None:
if stream.pending is not None:
call["update"] = stream.pending.update
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, context, **call
)
@@ -208,7 +220,8 @@ class StageCommunicator:
hidden_states = entry.handoff(
hidden_states, forward_batch, self._layer.qkv_latent_func
)
return hidden_states, residual
stream.write(residual)
return hidden_states, stream
class LayerFusions(Protocol):
@@ -231,6 +244,7 @@ class LayerCommunicator:
add_to_output = staticmethod(add_to_output)
buffer = staticmethod(buffer)
fold = staticmethod(fold)
written = staticmethod(written)
finish_layer_stack = staticmethod(finish_layer_stack)
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
@@ -602,15 +616,24 @@ class LayerCommunicator:
storage = buffer(hidden_states)
if storage is not None and storage.shape[0] == 0:
return hidden_states, None
hidden_states = reduce_output(hidden_states)
hidden_states = (
residual.complete(hidden_states)
if isinstance(residual, ResidualStream)
else reduce_output(hidden_states)
)
return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input)
def from_pp(self, tensors, *, allow_missing_residual: bool = False):
return from_pp(
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
hidden_states, residual = from_pp(
tensors,
residual_in_hidden=self._residual.ffn_update.at_producer,
allow_missing_residual=allow_missing_residual,
)
hidden_states, stream = ResidualStream.arrive(
hidden_states, residual, self._residual.ffn_update
)
forward_batch.residual_stream = stream
return hidden_states, stream
def prepare_attn_and_capture_last_layer_outputs(
self,
@@ -629,23 +652,26 @@ class LayerCommunicator:
post_residual_addition=post_residual_addition,
)
if captured_last_layer_outputs is not None:
residual_value = residual.residual
move = self.attn.entry(forward_batch).input_move
gathered_last_layer_output = (
residual
residual_value
if move is None
else move(
hidden_states=residual,
hidden_states=residual_value,
forward_batch=forward_batch,
context=self._context,
)
)
if (
gathered_last_layer_output is residual
gathered_last_layer_output is residual_value
# An accumulator that copies on append already holds a snapshot.
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
and not self._post_attn_residual_is_read_only(residual, forward_batch)
and not self._post_attn_residual_is_read_only(
residual_value, forward_batch
)
):
gathered_last_layer_output = residual.clone()
gathered_last_layer_output = residual_value.clone()
captured_last_layer_outputs.append(gathered_last_layer_output)
return hidden_states, residual
@@ -836,7 +862,18 @@ class LayerCommunicator:
if cache is not None:
self._context.cache = cache
return self.ffn.prepare(hidden_states, residual, forward_batch, steps)
if isinstance(residual, ResidualStream) and self.stage_edges is None:
# The paired boundary records the attention contribution once.
hidden_states = residual.leave(
hidden_states, self._residual.attention_update
)
return self.ffn.prepare(
hidden_states,
residual,
forward_batch,
steps,
update=self._residual.attention_update,
)
def maybe_prefetch_next_full_attention_kv(
self,
@@ -851,12 +888,28 @@ class LayerCommunicator:
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
return self._complete_ffn_output_now(
stream = residual if isinstance(residual, ResidualStream) else None
hidden_states, residual = self._complete_ffn_output_now(
hidden_states,
residual,
stream.residual if stream is not None else residual,
forward_batch=forward_batch,
dp_step=self._postprocess_dp_step(forward_batch),
)
return self._leave_ffn_output(
hidden_states,
residual,
stream,
self._batch_steps(forward_batch).ffn_output.update,
)
@staticmethod
def _leave_ffn_output(hidden_states, residual, stream, update):
if stream is None:
return hidden_states, residual
stream.residual = residual
if update.at_producer:
return stream.write(hidden_states), stream
return stream.leave(hidden_states, update), stream
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
@@ -1037,10 +1090,15 @@ class LayerCommunicator:
to the rows this FFN needs and its residual's rows."""
rows, residual_rows, _ = source._branch_rows(forward_batch)
to, residual_to, _ = self._branch_rows(forward_batch)
return (
move_rows(hidden_states, rows, to, forward_batch),
move_rows(residual, residual_rows, residual_to, forward_batch),
)
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
if stream.pending is not None:
raise RuntimeError("a branch must start from a prepared stage input")
residual = stream.residual
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
forward_batch.residual_stream = ResidualStream(residual)
return hidden_states, forward_batch.residual_stream
def branch_output(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
@@ -1064,9 +1122,19 @@ class LayerCommunicator:
this layer hands on."""
_, _, rows = source._branch_rows(forward_batch)
_, _, to = self._branch_rows(forward_batch)
stream = residual if isinstance(residual, ResidualStream) else None
update = None
if stream is not None:
stream.check(hidden_states)
update = stream.pending.update
hidden_states, residual = stream.finish(hidden_states)
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, rows, to, forward_batch)
return contribution + hidden_states, residual
output = contribution + hidden_states
if stream is not None:
stream.write(residual)
return stream.leave(output, update), stream
return output, residual
def _select_ffn_exit_fusions(
self,
@@ -1203,10 +1271,11 @@ class MixerExit:
its input), and when it may leave it and the fused kernel takes it into the
next input norm."""
__slots__ = ("skips_reduction", "_hands_on", "_scope")
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update")
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
produced = communicator._batch_steps(forward_batch).ffn_output
self._update = produced.update
self._hands_on = (
produced.leaves_for_next_layer
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
@@ -1221,13 +1290,15 @@ class MixerExit:
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor
) -> Union[torch.Tensor, UnreducedOutput]:
def finish(self, hidden_states: torch.Tensor, residual=None):
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
if self._hands_on:
return UnreducedOutput(hidden_states, group=get_parallel().tp_group)
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
return residual.leave(hidden_states, self._update), residual
return hidden_states
@@ -1243,6 +1314,7 @@ class FfnExit:
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_complete",
"_update",
"_scope",
)
@@ -1254,6 +1326,7 @@ class FfnExit:
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
self._update = communicator._batch_steps(forward_batch).ffn_output.update
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
@@ -1271,11 +1344,17 @@ class FfnExit:
self, hidden_states: torch.Tensor, residual: torch.Tensor
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input, or postprocess."""
stream = residual_batch.current(self.forward_batch)
if stream is not None:
residual = stream.residual
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
return hidden_states, residual
return self._complete(hidden_states, residual)
else:
hidden_states, residual = self._complete(hidden_states, residual)
return self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update
)
class MHCLayerCommunicator(LayerCommunicator):
+34 -4
View File
@@ -320,6 +320,7 @@ def _mlp_input_dp_partial(
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: one rank adds the residual, the gather sums it, then read
the input from the sum. With ``places_cp_shards`` each CP rank puts its
@@ -384,6 +385,22 @@ def _tp_all_gather_scattered_rows(
return output
def _dispatch_consumer(
hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call
):
if residual is None:
# Enter/read-only paths do not execute an update.
prepare = paths[False]
else:
try:
prepare = paths[update.adds_plainly]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
return prepare(
hidden_states, residual, forward_batch, norm, context, update=update, **call
)
def _consumer_step(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
@@ -392,8 +409,12 @@ def _consumer_step(
context: CommunicateContext,
*,
step: Callable,
adds_plainly: bool,
carried_fusions: Tuple[Callable, ...],
owes_by_construction: bool = False,
written_step: Optional[Callable] = None,
written: bool = False,
update: StageUpdate = ADD,
**call,
):
"""A boundary's half into a stage: complete what the value carries, a sum
@@ -404,6 +425,10 @@ def _consumer_step(
that owes its sum by construction (``owes_by_construction``) is that sum's
only carrier. ``call`` is what the stage's read takes (the attention's
``quant_format`` and ``post_residual_addition``)."""
if residual is not None and update.adds_plainly != adds_plainly:
raise RuntimeError("producer update does not match the boundary's capability")
if written and written_step is not None:
step = written_step
if not isinstance(hidden_states, torch.Tensor):
owed = hidden_states
if residual is None:
@@ -429,7 +454,9 @@ def _consumer_step(
hidden_states = reduce_output(owed)
else:
hidden_states = owed.partial
return step(hidden_states, residual, forward_batch, norm, context, **call)
return step(
hidden_states, residual, forward_batch, norm, context, update=update, **call
)
def _read_input(
@@ -442,7 +469,7 @@ def _read_input(
layer_input: Optional[Callable],
enters_stack: bool,
read: StageRead,
update: StageUpdate,
update: StageUpdate = ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
@@ -535,6 +562,7 @@ def _mlp_input_residual_into_sum(
context: CommunicateContext,
*,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context, read
@@ -565,12 +593,13 @@ def _mlp_input_gather_attention_cp(
context: CommunicateContext,
*,
gather: Callable,
update: StageUpdate = ADD,
):
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context
hidden_states, residual, forward_batch, layernorm, context, update=update
)
return dsa_cp_gather_hidden_states(hidden_states), residual
@@ -583,6 +612,7 @@ def _mlp_input_gather_moe_cp(
context: CommunicateContext,
*,
gather: Callable,
update: StageUpdate = ADD,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
@@ -596,7 +626,7 @@ def _mlp_input_gather_moe_cp(
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context
hidden_states, residual, forward_batch, layernorm, context, update=update
)
rows = moe_cp_gathered_rows(forward_batch)
@@ -31,6 +31,9 @@ class StageUpdate(Protocol):
# The stage writes its output into the residual itself at its end, instead
# of leaving that to the next stage's read.
at_producer: bool
# Its parameters and state remain valid after leaving the producer layer.
# Stateful or offloaded implementations must not opt in without that guarantee.
can_defer_across_layers: bool
def update(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the output written into it."""
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.output import (
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
@@ -32,6 +33,8 @@ def buffer(
"""Storage that can be reused after prepare consumes the input. A finalize
handoff has no reusable layer-output tensor. This does not complete or read
the value held in the storage."""
if isinstance(hidden_states, OwedOutput):
return hidden_states.contribution.value
if isinstance(hidden_states, UnreducedOutput):
return hidden_states.partial
if isinstance(hidden_states, HandoffOutput):
@@ -42,18 +45,35 @@ def buffer(
def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple:
"""Complete the output before adding an extra contribution exactly once.
Leave its residual update for the next prepare call."""
hidden_states = reduce_output(hidden_states)
hidden_states = (
residual.complete(hidden_states)
if isinstance(residual, ResidualStream)
else reduce_output(hidden_states)
)
hidden_states.add_(extra)
return hidden_states, residual
def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]:
def fold(hidden_states, residual):
"""Finish a plain layer output and fold its residual into a complete value.
The following stage receives it with no outstanding residual addition."""
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
hidden_states, residual = stream.finish(hidden_states)
hidden_states = reduce_output(hidden_states)
if residual is not None:
hidden_states = hidden_states + residual
return hidden_states, None
return (
(stream.write(hidden_states), stream)
if stream is not None
else (hidden_states, None)
)
def written(hidden_states, forward_batch):
"""Re-enter after a computation that already updated the full residual."""
forward_batch.residual_stream = ResidualStream(hidden_states)
return hidden_states, forward_batch.residual_stream
def finish_layer_stack(
@@ -68,6 +88,13 @@ def finish_layer_stack(
pipeline rank, or any other consumer outside the layers. A final norm
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
if isinstance(residual, ResidualStream):
stream = forward_batch.residual_stream
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff)
forward_batch.residual_stream = None
return output
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
return hidden_states, residual
return reduce_output(hidden_states), residual
@@ -99,6 +126,8 @@ def snapshot(
) -> torch.Tensor:
"""Copy a complete output with its plain residual. A statically declared
sum is reduced on a copy; the main output and residual remain unchanged."""
if isinstance(residual, ResidualStream):
return residual.snapshot(hidden_states, group=group)
if group is not None:
hidden_states = group.all_reduce(hidden_states.clone())
return hidden_states.clone() if residual is None else hidden_states + residual
@@ -267,6 +267,7 @@ class Add:
adds_plainly = True
at_producer = False
can_defer_across_layers = True
def update(self, hidden_states, residual):
hidden_states += residual
@@ -0,0 +1,18 @@
"""Own one invocation's residual stream, including independent TBO children."""
from sglang.srt.layers.communicator.residual.stream import ResidualStream
def start(forward_batch):
"""Begin a fresh embedding-side invocation and return its stream."""
stream = ResidualStream()
forward_batch.residual_stream = stream
return stream
def current(forward_batch):
"""Return the invocation's authoritative stream while its stack is active."""
stream = forward_batch.residual_stream
if stream is None:
raise RuntimeError("start the layer stack before entering a stage")
return stream
@@ -143,6 +143,7 @@ class _AttentionUpdate:
adds_plainly = False
at_producer = False
can_defer_across_layers = False
def __init__(self, state: MHCState):
self.state = state
@@ -185,6 +186,7 @@ class _FfnUpdate:
adds_plainly = False
at_producer = True
can_defer_across_layers = False
def __init__(self, state: MHCState):
self.state = state
@@ -0,0 +1,168 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""One forward's residual and its producer's outstanding contribution."""
from typing import Callable, Optional, Union
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
from sglang.srt.layers.communicator.residual import StageUpdate
class CarriedSum(msgspec.Struct, frozen=True):
"""Work left by the producer, including a move back to local token rows."""
group: Optional[GroupCoordinator] = None
reduce_and_redistribute: Optional[Callable] = None
def complete(self, value):
if self.reduce_and_redistribute is not None:
return self.reduce_and_redistribute(value)
return self.group.all_reduce(value)
class Contribution(msgspec.Struct):
"""The sole owner of an output's value, update and remaining work."""
value: Optional[torch.Tensor]
update: StageUpdate
owed: Union[CarriedSum, HandoffOutput, None] = None
def for_boundary(self):
# These forms are private inputs to the existing fused-kernel adapters.
if isinstance(self.owed, CarriedSum):
return UnreducedOutput(
self.value, self.owed.group, self.owed.reduce_and_redistribute
)
if isinstance(self.owed, HandoffOutput):
return self.owed
return self.value
def complete(self):
if isinstance(self.owed, CarriedSum):
value = self.owed.complete(self.value)
elif isinstance(self.owed, HandoffOutput):
value = self.owed.complete()
else:
return self.value
self.value, self.owed = value, None
return value
class OwedOutput(msgspec.Struct, frozen=True):
"""Opaque model-facing handle. Only its boundary may read the contribution."""
contribution: Contribution
class ResidualStream:
"""Uninitialized, written, or holding one pending producer contribution.
Each forward or TBO microbatch owns its stream. A prepare consumes its
pending contribution once; completing a sum for capture leaves the update
pending. No layout or batch selection is stored on the value.
"""
__slots__ = ("residual", "pending")
def __init__(self, residual=None):
self.residual = residual
self.pending = None
@classmethod
def arrive(cls, hidden, residual, update):
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
stream = cls(residual)
if residual is None:
return stream.write(hidden), stream
return stream.leave(hidden, update), stream
def write(self, residual):
self.residual = residual
self.pending = None
return residual
def leave(self, output, update):
if self.pending is not None:
raise RuntimeError("cannot replace an unconsumed producer contribution")
if isinstance(output, UnreducedOutput):
contribution = Contribution(
output.partial,
update,
CarriedSum(output.group, output.reduce_and_redistribute),
)
elif isinstance(output, HandoffOutput):
contribution = Contribution(None, update, output)
else:
contribution = Contribution(output, update)
self.pending = contribution
return OwedOutput(contribution) if contribution.owed is not None else output
def check(self, hidden):
pending = self.pending
if pending is None:
if isinstance(hidden, OwedOutput):
raise RuntimeError("an owed handle has no contribution in this stream")
if self.residual is not None and hidden is not self.residual:
raise RuntimeError(
"written residual does not match the supplied output"
)
return
if pending.owed is not None:
valid = isinstance(hidden, OwedOutput) and hidden.contribution is pending
else:
valid = hidden is pending.value
if not valid:
raise RuntimeError(
"output does not belong to this residual stream; "
"change layer outputs through boundary accessors"
)
def input(self, hidden):
self.check(hidden)
if self.pending is not None:
return self.pending.for_boundary(), self.residual
# An initialized, written stream needs read only. An uninitialized
# stream still uses the first stage's enter operation.
return hidden, None
def complete(self, hidden):
self.check(hidden)
return self.pending.complete() if self.pending is not None else hidden
def snapshot(self, hidden, *, group=None):
self.check(hidden)
pending = self.pending
if pending is None:
return hidden.clone()
if pending.owed is None:
value = pending.value
elif isinstance(pending.owed, CarriedSum):
value = pending.owed.complete(pending.value.clone())
else:
raise NotImplementedError("a finalize handoff requires main-output capture")
if group is not None:
value = group.all_reduce(value.clone())
return value.clone() if self.residual is None else value + self.residual
def finish(self, hidden, *, takes_handoff=False):
self.check(hidden)
if self.pending is None:
return hidden, None
if takes_handoff and isinstance(self.pending.owed, HandoffOutput):
return self.pending.owed, self.residual
return self.pending.complete(), self.residual
@@ -71,6 +71,7 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
@@ -564,6 +565,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_prefill_cuda_graph_max_prefix_len: int = 0
global_forward_mode: Optional[ForwardMode] = None
# Owned only while this invocation executes its local decoder stack.
residual_stream: Optional[ResidualStream] = None
# For two-batch overlap
tbo_split_seq_index: Optional[int] = None
+3 -2
View File
@@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -814,12 +815,12 @@ class BailingMoEModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
@@ -19,6 +19,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -980,12 +981,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = inputs_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
total_num_layers = self.end_layer - self.start_layer
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
@@ -26,6 +26,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
@@ -161,7 +162,7 @@ class BailingMoEModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
if self.is_hybrid:
device = input_ids.device
zero_allocator = BumpAllocator(
+3 -2
View File
@@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -1339,12 +1340,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
total_num_layers = self.end_layer - self.start_layer
device = hidden_states.device
+2 -1
View File
@@ -28,6 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -215,7 +216,7 @@ class DeepseekModelNextN(nn.Module):
else:
hidden_states = self.eh_proj(eh_input)
residual = None
residual = residual_batch.start(forward_batch)
index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual, topk_indices = self.decoder(
+3 -2
View File
@@ -76,6 +76,7 @@ from sglang.srt.layers.communicator import (
from sglang.srt.layers.communicator.adapters.context_parallel import (
maybe_prefetch_next_full_attention_kv,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.dcp.planner import (
prepare_decode_context_parallel_metadata,
@@ -3027,13 +3028,13 @@ class DeepseekV2Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
initial_topk_indices = None
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
if not self.pp_group.is_first_rank:
@@ -62,6 +62,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -1765,12 +1766,12 @@ class Dots3Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
@@ -8,6 +8,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
@@ -113,7 +114,7 @@ class Dot3NoteModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = head.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
+2 -1
View File
@@ -12,6 +12,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -421,7 +422,7 @@ class FalconH1Model(nn.Module):
else:
hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier
residual = None
residual = residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
+3 -2
View File
@@ -24,6 +24,7 @@ from transformers import PretrainedConfig
import sglang.srt.models.deepseek_v2 as deepseek_v2
from sglang.srt.configs.gigachat35 import GigaChat35Config
from sglang.srt.layers.communicator import get_attn_tp_context
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -558,12 +559,12 @@ class GigaChat35Model(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
total_num_layers = self.end_layer - self.start_layer
zero_allocator = BumpAllocator(
+2 -1
View File
@@ -18,6 +18,7 @@ from typing import Iterable, Optional
import torch
from torch import nn
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -106,7 +107,7 @@ class GigaChat35ModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
hidden_states, residual = self.decoder(
positions=positions,
hidden_states=hidden_states,
+3 -2
View File
@@ -43,6 +43,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
@@ -1084,12 +1085,12 @@ class Glm4MoeModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
+3 -2
View File
@@ -38,6 +38,7 @@ from sglang.srt.layers.communicator import (
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
@@ -776,12 +777,12 @@ class Glm4MoeLiteModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
@@ -22,6 +22,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -114,7 +115,7 @@ class Glm4MoeLiteModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
+2 -1
View File
@@ -22,6 +22,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -100,7 +101,7 @@ class Glm4MoeModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual
+3 -2
View File
@@ -32,6 +32,7 @@ from sglang.srt.layers.communicator import (
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelBatchedLinear,
@@ -1100,12 +1101,12 @@ class Glm5NextModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
device = hidden_states.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
+3 -2
View File
@@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -700,12 +701,12 @@ class GptOssModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
+2 -1
View File
@@ -11,6 +11,7 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusTextConfig,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
@@ -762,7 +763,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = None
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
for layer_idx, layer in enumerate(self.layers):
hidden_states, residual = layer(
+3 -2
View File
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -581,12 +582,12 @@ class LagunaModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
+3 -2
View File
@@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -746,12 +747,12 @@ class LLaDA2MoeModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
+35 -1
View File
@@ -26,6 +26,7 @@ from torch import nn
from transformers import Llama4TextConfig
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -518,7 +519,7 @@ class Llama4Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
for i in range(len(self.layers)):
if i in self.layers_to_capture:
@@ -561,6 +562,39 @@ class Llama4ForCausalLM(LlamaForCausalLM):
):
super().__init__(config, quant_config, prefix)
@torch.no_grad()
def forward_split_prefill(
self, input_ids, positions, forward_batch, split_interval, input_embeds=None
):
start, end = split_interval
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
forward_batch.hidden_states = (
self.model.embed_tokens(input_ids)
if input_embeds is None
else input_embeds
)
for layer in self.model.layers[start:end]:
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
)
if end != self.model.config.num_hidden_layers:
return None
hidden, residual = LayerCommunicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
)
forward_batch.residual = None
if residual is None:
forward_batch.hidden_states = self.model.norm(hidden)
else:
forward_batch.hidden_states, _ = self.model.norm(hidden, residual)
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
)
def get_input_embeddings(self):
return self.model.embed_tokens
+2 -1
View File
@@ -52,6 +52,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -624,7 +625,7 @@ class LongcatFlashModel(nn.Module):
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
topk_indices = None
@@ -42,6 +42,7 @@ from sglang.srt.configs import LongcatFlashConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -269,7 +270,7 @@ class LongcatFlashModelNextN(nn.Module):
)
)
residual = None
residual = residual_batch.start(forward_batch)
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual, zero_allocator
+3 -2
View File
@@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1037,12 +1038,12 @@ class MiMoV2Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
if forward_batch.can_run_tbo and not self.layers_to_capture:
+2 -1
View File
@@ -26,6 +26,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -223,7 +224,7 @@ class MiMoV2ModelNextN(nn.Module):
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=None,
residual=residual_batch.start(forward_batch),
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
+3 -2
View File
@@ -34,6 +34,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
@@ -1141,12 +1142,12 @@ class MiniMaxM2Model(nn.Module):
hidden_states = self.get_input_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
if forward_batch.can_run_tbo:
+3 -2
View File
@@ -40,6 +40,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1476,12 +1477,12 @@ class MiniMaxM3Model(nn.Module):
hidden_states = embeds(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
if forward_batch.can_run_tbo:
+5 -1
View File
@@ -23,6 +23,7 @@ from sglang.srt.layers.attention.vision import (
prepare_vision_attention_metadata,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -1109,7 +1110,7 @@ class MossVLTextModel(nn.Module):
vision_position_ids: Optional[torch.Tensor] = None,
) -> torch.Tensor:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual = residual_batch.start(forward_batch)
for decoder_layer in self.layers:
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
@@ -1126,6 +1127,9 @@ class MossVLTextModel(nn.Module):
positions=positions,
vision_position_ids=vision_position_ids,
)
hidden_states, residual = LayerCommunicator.written(
hidden_states, forward_batch
)
elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer):
hidden_states, residual = decoder_layer(
positions=positions,
+4 -3
View File
@@ -31,6 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -478,7 +479,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
hidden_states = self._forward_mixer(
hidden_states, forward_batch, mixer_exit.skips_reduction
)
return mixer_exit.finish(hidden_states), residual
return mixer_exit.finish(hidden_states, residual)
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
@@ -732,12 +733,12 @@ class NemotronHModel(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
@@ -18,6 +18,7 @@ import torch
from torch import nn
from sglang.srt.configs import NemotronHConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
@@ -304,6 +305,8 @@ class NemotronHMultiTokenPredictor(nn.Module):
residual = None
for i in range(self.pattern_len):
if residual is None:
residual = residual_batch.start(forward_batch)
hidden_states, residual = self.layers[str(i)](
inputs_embeds=inputs_embeds,
hidden_states=hidden_states,
+5 -2
View File
@@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1175,12 +1176,12 @@ class Qwen2MoeModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
if forward_batch.can_run_tbo:
@@ -1306,6 +1307,7 @@ class Qwen2MoeForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -1332,6 +1334,7 @@ class Qwen2MoeForCausalLM(nn.Module):
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
+62 -1
View File
@@ -1,11 +1,13 @@
# Adapted from qwen2.py
import logging
from typing import Any, Dict, Iterable, List, Optional, Tuple
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import torch
from torch import nn
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -446,6 +448,63 @@ class Qwen3Model(Qwen2Model):
alt_stream=alt_stream,
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> Union[torch.Tensor, PPProxyTensors]:
if self.pp_group.is_first_rank:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(
residual_access.snapshot(hidden_states, residual)
)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
hidden_states, residual = residual_access.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) == 0:
return hidden_states
return hidden_states, aux_hidden_states
class Qwen3ForCausalLM(nn.Module):
# BitandBytes specific attributes
@@ -554,6 +613,7 @@ class Qwen3ForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -578,6 +638,7 @@ class Qwen3ForCausalLM(nn.Module):
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
+5 -4
View File
@@ -30,8 +30,6 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
qwen3_5_gdn_prefill_projection_views,
)
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
# Configs
from sglang.srt.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5MoeConfig,
@@ -47,6 +45,9 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
# Configs
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1817,12 +1818,12 @@ class Qwen3_5ForCausalLM(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
# Pass through decoder layers
+3
View File
@@ -30,6 +30,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
QKVParallelLinear,
@@ -1084,6 +1085,7 @@ class Qwen3MoeForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -1110,6 +1112,7 @@ class Qwen3MoeForCausalLM(nn.Module):
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
+2 -1
View File
@@ -16,6 +16,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -929,7 +930,7 @@ class Qwen3NextModel(nn.Module):
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
for i in range(len(self.layers)):
layer = self.layers[i]
+3 -2
View File
@@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -1204,12 +1205,12 @@ class Qwen3LLMModel(Qwen3Model):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for layer_idx, layer in enumerate(
+3 -2
View File
@@ -24,6 +24,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import get_layer_id
@@ -94,12 +95,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
for layer_idx, layer in enumerate(
+7 -4
View File
@@ -24,6 +24,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1151,12 +1152,12 @@ class SarvamMLAModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
@@ -1267,7 +1268,7 @@ class SarvamMLAForCausalLM(nn.Module):
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
forward_batch.hidden_states = input_embeds
forward_batch.residual = None
forward_batch.residual = residual_batch.start(forward_batch)
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -1291,6 +1292,7 @@ class SarvamMLAForCausalLM(nn.Module):
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
@@ -1455,7 +1457,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
forward_batch.hidden_states = self.model.word_embeddings(input_ids)
else:
forward_batch.hidden_states = input_embeds
forward_batch.residual = None
forward_batch.residual = residual_batch.start(forward_batch)
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
@@ -1479,6 +1481,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
return self.logits_processor(
+5 -2
View File
@@ -12,6 +12,7 @@ from transformers import PretrainedConfig
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -403,12 +404,14 @@ class SDARModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True)
].layer_communicator.from_pp(
pp_proxy_tensors, forward_batch, allow_missing_residual=True
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
+3 -2
View File
@@ -17,6 +17,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -469,12 +470,12 @@ class SDARMoeModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
+2 -1
View File
@@ -23,6 +23,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -471,7 +472,7 @@ class Step3TextModel(nn.Module):
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
+3 -2
View File
@@ -12,6 +12,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -697,12 +698,12 @@ class Step3p5Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
+2 -1
View File
@@ -6,6 +6,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -113,7 +114,7 @@ class Step3p5AMultiTokenPredictor(nn.Module):
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=None,
residual=residual_batch.start(forward_batch),
)
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
+3 -2
View File
@@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import (
LayerFacts,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -1637,12 +1638,12 @@ class XllmModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual = residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors)
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
for i in range(self.start_layer, self.end_layer):
ctx = (