[Fix] Return an FFN's output to the attention rows at a pipeline handoff (#42892)

This commit is contained in:
Cheng Wan
2026-10-09 01:25:05 -07:00
committed by GitHub
parent b2402ccad6
commit 48a6c0a5d0
8 changed files with 177 additions and 20 deletions
@@ -142,6 +142,8 @@ class StagePlan:
is_branch: Branch reusing an already-read input; requires branch_input
instead of a normal prepare to avoid repeating the read/update.
terminal: Whether the stage ends the model's layer stack.
writes_at_handoff: Whether the stage, an FFN handing off to another
pipeline rank, writes its output into the residual at its exit.
finishes_directly: Attention can publish its output with finish instead of
an exit scope and output transport.
qkv_latent_func: Optional hook for prepared attention input.
@@ -164,6 +166,7 @@ class StagePlan:
enters_stack=False,
is_branch=False,
terminal=False,
writes_at_handoff=False,
finishes_directly=False,
qkv_latent_func=None,
fusions=None,
@@ -217,6 +220,7 @@ class StagePlan:
output_move=out.output_move,
output_move_completes_sum=out.output_move_completes_sum,
returns_over_dp=out.returns_over_dp,
writes_at_handoff=writes_at_handoff,
)
@property
@@ -304,6 +308,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
enters_stack=incoming.producer is None,
is_branch=declaration.prepared_from is not None,
terminal=declaration.terminal,
writes_at_handoff=declaration.writes_at_handoff,
finishes_directly=declaration.kind is StageKind.ATTENTION
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
**options,
@@ -233,6 +233,8 @@ class StagePath(msgspec.Struct, frozen=True):
batch by the attention-DP exit path.
output_move_completes_sum: Whether that move also reduces the output.
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
writes_at_handoff: Whether the exit completes the output and writes it
into the residual, for an FFN that hands off to another pipeline rank.
"""
entry: EntryPath
@@ -242,6 +244,7 @@ class StagePath(msgspec.Struct, frozen=True):
output_move_completes_sum: bool = False
returns_over_dp: bool = False
writes_at_handoff: bool = False
class StageKind(Enum):
@@ -194,8 +194,10 @@ class ExitPolicy:
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
)
if not steps.output.may_defer_to_next and not (
self.plan.terminal and defer_moe_finalize
# An FFN that writes its output at a pipeline handoff completes it here.
if steps.writes_at_handoff or (
not steps.output.may_defer_to_next
and not (self.plan.terminal and defer_moe_finalize)
):
return ExitDecision(
defer_moe_finalize=False,
@@ -279,7 +281,7 @@ class ExitPolicy:
forward_batch=forward_batch,
)
update = steps.output.update
if residual is not None and update.applied_at_exit:
if residual is not None and (update.applied_at_exit or steps.writes_at_handoff):
hidden_states = update.update(hidden_states, residual)
residual = None
return hidden_states, residual
@@ -77,6 +77,16 @@ def _row_layouts(variant):
return axes, attention, local, Layout.sharded_over(axis_sizes=axes)
def _ffn_on_rank_rows(sparse, dense_tp_size) -> bool:
"""Whether an FFN computes on this rank's own rows (its attention-TP slice
of them, or an unpadded batch's rows) rather than on the attention's."""
if sparse:
return is_moe_input_scattered_across_dp_ranks()
if dense_tp_size is None:
return is_dense_ffn_fully_dp()
return dense_tp_size == 1
def _resolve_ffn(
variant,
*,
@@ -100,11 +110,7 @@ def _resolve_ffn(
use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output
cp_shards = _prefill_cp_shards_tokens()
axes, attention, local, full = _row_layouts(variant)
on_rank_rows = (
is_moe_input_scattered_across_dp_ranks()
if sparse
else (is_dense_ffn_fully_dp() if dense_tp_size is None else dense_tp_size == 1)
)
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
if parallel.attn_cp_size > 1 and sparse:
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
on_cp_shards = (
@@ -214,6 +220,8 @@ class StageDeclaration:
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
exit_rows: Required FFN output rows at the layer or branch exit.
writes_at_handoff: Whether this FFN writes its output into the residual
at its exit because the next stage is on another pipeline rank.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
prepared_from: Declaration whose already-read input a branch reuses.
@@ -233,6 +241,7 @@ class StageDeclaration:
tp_group: SumGroup = SumGroup.ATTN_TP
dense_tp_size: Optional[int] = None
exit_rows: Optional[ExitRows] = None
writes_at_handoff: bool = False
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -722,10 +731,28 @@ class _Chain:
__slots__ = ("previous", "pending")
def __init__(self, previous):
self.previous = _detached(previous)
self.previous = _detached(_handed_off(previous))
self.pending = None
def _handed_off(declaration):
"""A stage whose output crosses to another pipeline rank: an FFN leaves it
on the attention's rows, which is what the handoff carries and the next
rank's first stage reads, even where it would otherwise stay on this
rank's attention-TP slice of them. An FFN on its own rows also writes its
output into the residual there, on every batch, so the handoff carries the
written stream and the receiver knows it from this declaration."""
if declaration is None or declaration.kind is not StageKind.FFN:
return declaration
return replace(
declaration,
exit_rows=ExitRows.ATTENTION,
writes_at_handoff=_ffn_on_rank_rows(
declaration.sparse, declaration.dense_tp_size
),
)
def _detached(declaration):
"""The declaration a following append extends, without its own history.
@@ -741,6 +768,10 @@ def _detached(declaration):
def _bind_stack(appends, *, previous, following):
"""Bind every appended stage, in order, and fill in the boundaries each
append returned."""
if following is not None:
# The last stage hands off to the next rank.
last = next(a for a in reversed(appends) if a.prepared_from is None)
last.declarations[-1] = _handed_off(last.declarations[-1])
chain = _Chain(previous)
# A returned declaration's boundary as bound, for the branches that read it.
sources = {}
@@ -223,12 +223,12 @@ class StageBoundary:
Tensor or owed handle for prepare. A producer-written residual and a
declared partial sum are reconstructed from the incoming contract.
"""
previous = self.declaration.previous
hidden_states, residual = from_pp(
tensors,
residual_in_hidden=(
self.declaration.previous is not None
and self.declaration.previous.update.applied_at_exit
),
# The producer's declaration says whether the stream arrives written.
residual_in_hidden=previous is not None
and (previous.update.applied_at_exit or previous.writes_at_handoff),
allow_missing_residual=allow_missing_residual,
)
declared_sum = self.entry(forward_batch).declared_sum
+18 -6
View File
@@ -1,7 +1,9 @@
"""Small fixtures around production boundaries; no parallel layout recipes."""
from types import SimpleNamespace
from unittest import mock
from sglang.srt.layers.layer_boundary import factories
from sglang.srt.layers.layer_boundary.construction import StagePlan
from sglang.srt.layers.layer_boundary.factories import (
append_stages,
@@ -21,15 +23,25 @@ def build_stages(*stages, previous=None, terminal=False):
*stages: Items of (declaration, norm) or (declaration, norm, options),
as for append_stages.
previous: Declaration of the stage before the first one, standing for
a layer the test does not build; None starts the stack.
a layer on the same rank that the test does not build; None starts
the stack.
terminal: Whether the last stage ends the model's layer stack;
otherwise the next layer's attention follows it.
otherwise the next layer's attention follows it on the same rank.
"""
with layer_stack(
previous_layers=(
[lambda: append_stages((previous, None))] if previous is not None else []
# The stack's neighbour hooks build layers another pipeline rank holds;
# here they stand for this rank's, so nothing is handed off to them.
with (
mock.patch.object(factories, "_handed_off", lambda declaration: declaration),
layer_stack(
previous_layers=(
[lambda: append_stages((previous, None))]
if previous is not None
else []
),
next_layers=(
[] if terminal else [lambda: append_stages((declare_attn(), None))]
),
),
next_layers=[] if terminal else [lambda: append_stages((declare_attn(), None))],
):
return append_stages(*stages)