[Fix] Return an FFN's output to the attention rows at a pipeline handoff (#42892)
This commit is contained in:
@@ -142,6 +142,8 @@ class StagePlan:
|
||||
is_branch: Branch reusing an already-read input; requires branch_input
|
||||
instead of a normal prepare to avoid repeating the read/update.
|
||||
terminal: Whether the stage ends the model's layer stack.
|
||||
writes_at_handoff: Whether the stage, an FFN handing off to another
|
||||
pipeline rank, writes its output into the residual at its exit.
|
||||
finishes_directly: Attention can publish its output with finish instead of
|
||||
an exit scope and output transport.
|
||||
qkv_latent_func: Optional hook for prepared attention input.
|
||||
@@ -164,6 +166,7 @@ class StagePlan:
|
||||
enters_stack=False,
|
||||
is_branch=False,
|
||||
terminal=False,
|
||||
writes_at_handoff=False,
|
||||
finishes_directly=False,
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
@@ -217,6 +220,7 @@ class StagePlan:
|
||||
output_move=out.output_move,
|
||||
output_move_completes_sum=out.output_move_completes_sum,
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -304,6 +308,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
enters_stack=incoming.producer is None,
|
||||
is_branch=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
writes_at_handoff=declaration.writes_at_handoff,
|
||||
finishes_directly=declaration.kind is StageKind.ATTENTION
|
||||
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
|
||||
**options,
|
||||
|
||||
@@ -233,6 +233,8 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
batch by the attention-DP exit path.
|
||||
output_move_completes_sum: Whether that move also reduces the output.
|
||||
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
|
||||
writes_at_handoff: Whether the exit completes the output and writes it
|
||||
into the residual, for an FFN that hands off to another pipeline rank.
|
||||
"""
|
||||
|
||||
entry: EntryPath
|
||||
@@ -242,6 +244,7 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
output_move_completes_sum: bool = False
|
||||
|
||||
returns_over_dp: bool = False
|
||||
writes_at_handoff: bool = False
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
|
||||
@@ -194,8 +194,10 @@ class ExitPolicy:
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
)
|
||||
if not steps.output.may_defer_to_next and not (
|
||||
self.plan.terminal and defer_moe_finalize
|
||||
# An FFN that writes its output at a pipeline handoff completes it here.
|
||||
if steps.writes_at_handoff or (
|
||||
not steps.output.may_defer_to_next
|
||||
and not (self.plan.terminal and defer_moe_finalize)
|
||||
):
|
||||
return ExitDecision(
|
||||
defer_moe_finalize=False,
|
||||
@@ -279,7 +281,7 @@ class ExitPolicy:
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
update = steps.output.update
|
||||
if residual is not None and update.applied_at_exit:
|
||||
if residual is not None and (update.applied_at_exit or steps.writes_at_handoff):
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -77,6 +77,16 @@ def _row_layouts(variant):
|
||||
return axes, attention, local, Layout.sharded_over(axis_sizes=axes)
|
||||
|
||||
|
||||
def _ffn_on_rank_rows(sparse, dense_tp_size) -> bool:
|
||||
"""Whether an FFN computes on this rank's own rows (its attention-TP slice
|
||||
of them, or an unpadded batch's rows) rather than on the attention's."""
|
||||
if sparse:
|
||||
return is_moe_input_scattered_across_dp_ranks()
|
||||
if dense_tp_size is None:
|
||||
return is_dense_ffn_fully_dp()
|
||||
return dense_tp_size == 1
|
||||
|
||||
|
||||
def _resolve_ffn(
|
||||
variant,
|
||||
*,
|
||||
@@ -100,11 +110,7 @@ def _resolve_ffn(
|
||||
use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output
|
||||
cp_shards = _prefill_cp_shards_tokens()
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
on_rank_rows = (
|
||||
is_moe_input_scattered_across_dp_ranks()
|
||||
if sparse
|
||||
else (is_dense_ffn_fully_dp() if dense_tp_size is None else dense_tp_size == 1)
|
||||
)
|
||||
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
|
||||
on_cp_shards = (
|
||||
@@ -214,6 +220,8 @@ class StageDeclaration:
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
writes_at_handoff: Whether this FFN writes its output into the residual
|
||||
at its exit because the next stage is on another pipeline rank.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
@@ -233,6 +241,7 @@ class StageDeclaration:
|
||||
tp_group: SumGroup = SumGroup.ATTN_TP
|
||||
dense_tp_size: Optional[int] = None
|
||||
exit_rows: Optional[ExitRows] = None
|
||||
writes_at_handoff: bool = False
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -722,10 +731,28 @@ class _Chain:
|
||||
__slots__ = ("previous", "pending")
|
||||
|
||||
def __init__(self, previous):
|
||||
self.previous = _detached(previous)
|
||||
self.previous = _detached(_handed_off(previous))
|
||||
self.pending = None
|
||||
|
||||
|
||||
def _handed_off(declaration):
|
||||
"""A stage whose output crosses to another pipeline rank: an FFN leaves it
|
||||
on the attention's rows, which is what the handoff carries and the next
|
||||
rank's first stage reads, even where it would otherwise stay on this
|
||||
rank's attention-TP slice of them. An FFN on its own rows also writes its
|
||||
output into the residual there, on every batch, so the handoff carries the
|
||||
written stream and the receiver knows it from this declaration."""
|
||||
if declaration is None or declaration.kind is not StageKind.FFN:
|
||||
return declaration
|
||||
return replace(
|
||||
declaration,
|
||||
exit_rows=ExitRows.ATTENTION,
|
||||
writes_at_handoff=_ffn_on_rank_rows(
|
||||
declaration.sparse, declaration.dense_tp_size
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _detached(declaration):
|
||||
"""The declaration a following append extends, without its own history.
|
||||
|
||||
@@ -741,6 +768,10 @@ def _detached(declaration):
|
||||
def _bind_stack(appends, *, previous, following):
|
||||
"""Bind every appended stage, in order, and fill in the boundaries each
|
||||
append returned."""
|
||||
if following is not None:
|
||||
# The last stage hands off to the next rank.
|
||||
last = next(a for a in reversed(appends) if a.prepared_from is None)
|
||||
last.declarations[-1] = _handed_off(last.declarations[-1])
|
||||
chain = _Chain(previous)
|
||||
# A returned declaration's boundary as bound, for the branches that read it.
|
||||
sources = {}
|
||||
|
||||
@@ -223,12 +223,12 @@ class StageBoundary:
|
||||
Tensor or owed handle for prepare. A producer-written residual and a
|
||||
declared partial sum are reconstructed from the incoming contract.
|
||||
"""
|
||||
previous = self.declaration.previous
|
||||
hidden_states, residual = from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=(
|
||||
self.declaration.previous is not None
|
||||
and self.declaration.previous.update.applied_at_exit
|
||||
),
|
||||
# The producer's declaration says whether the stream arrives written.
|
||||
residual_in_hidden=previous is not None
|
||||
and (previous.update.applied_at_exit or previous.writes_at_handoff),
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
declared_sum = self.entry(forward_batch).declared_sum
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Small fixtures around production boundaries; no parallel layout recipes."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.layers.layer_boundary import factories
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
append_stages,
|
||||
@@ -21,15 +23,25 @@ def build_stages(*stages, previous=None, terminal=False):
|
||||
*stages: Items of (declaration, norm) or (declaration, norm, options),
|
||||
as for append_stages.
|
||||
previous: Declaration of the stage before the first one, standing for
|
||||
a layer the test does not build; None starts the stack.
|
||||
a layer on the same rank that the test does not build; None starts
|
||||
the stack.
|
||||
terminal: Whether the last stage ends the model's layer stack;
|
||||
otherwise the next layer's attention follows it.
|
||||
otherwise the next layer's attention follows it on the same rank.
|
||||
"""
|
||||
with layer_stack(
|
||||
previous_layers=(
|
||||
[lambda: append_stages((previous, None))] if previous is not None else []
|
||||
# The stack's neighbour hooks build layers another pipeline rank holds;
|
||||
# here they stand for this rank's, so nothing is handed off to them.
|
||||
with (
|
||||
mock.patch.object(factories, "_handed_off", lambda declaration: declaration),
|
||||
layer_stack(
|
||||
previous_layers=(
|
||||
[lambda: append_stages((previous, None))]
|
||||
if previous is not None
|
||||
else []
|
||||
),
|
||||
next_layers=(
|
||||
[] if terminal else [lambda: append_stages((declare_attn(), None))]
|
||||
),
|
||||
),
|
||||
next_layers=[] if terminal else [lambda: append_stages((declare_attn(), None))],
|
||||
):
|
||||
return append_stages(*stages)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user