[Refactor] Record what a bound stage path does instead of reading it back (#43453)

This commit is contained in:
Cheng Wan
2026-10-10 12:40:30 -07:00
committed by GitHub
parent 0acd93c35c
commit 8bd7f2d178
11 changed files with 121 additions and 207 deletions
@@ -55,7 +55,6 @@ class AttentionInputs:
self.hidden_states_local = hidden_states
self.forward_batch = forward_batch
self.qkv_latent_func = qkv_latent_func
self.hidden_states_ = None
self.qkv_latent_ = None
# When True, hidden_states_local is already attn_tp-gathered upstream
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
@@ -72,14 +71,6 @@ class AttentionInputs:
self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch)
return self.qkv_latent_
def fetch_hidden_states(self):
if self.hidden_states_ is not None:
return self.hidden_states_
self.hidden_states_ = self.hidden_states_local
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch)
return self.hidden_states_
class AttnTpContext:
def __init__(self):
@@ -131,11 +122,6 @@ class AttnTpContext:
assert attn_inputs is not None
return attn_inputs.fetch_qkv_latent()
def fetch_hidden_states(self):
attn_inputs = get_forward().attn_inputs
assert attn_inputs is not None
return attn_inputs.fetch_hidden_states()
def clear_attn_inputs(self) -> None:
get_forward().set("attn_inputs", None)
@@ -55,7 +55,6 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.prepare import (
_attn_tp_reduce_scatter_update_read,
_attn_tp_slice_update_read,
_dispatch_by_update,
_dp_gather_sum_read,
_move_before_read,
_reduce_update_read,
@@ -116,6 +115,26 @@ class ExitMove(NamedTuple):
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
returns_over_dp: bool = False
# Whether output_move gathers the rows back over attention TP.
gathers_attn_tp: bool = False
class EntryStep(NamedTuple):
"""The steps chosen into a stage, with what they do that binding and
capture rely on."""
step: Callable
# Moves the input onto the stage's rows after the step, or None.
input_move: Optional[Callable] = None
# input_move gathers over attention TP with the stage's own gather.
input_gather_declared: bool = False
# input_move returns storage of its own on every path it takes, which a
# capture may keep without copying: the boundary's gather does; a stage's
# own gather may return a buffer it reuses.
input_retainable: bool = False
# The kernels the step tries for the sum over attention TP it completes
# without moving the residual.
ffn_fusions: Tuple[FfnInputFusion, ...] = ()
def input_rows(edge: EdgeContract) -> Layout:
@@ -203,8 +222,6 @@ def bind_entry(
Returns:
An EntryPath whose prepare accepts the actual update from the stream.
Multiple update capabilities select among preconstructed paths; a single
capability needs no runtime dispatcher.
"""
if edge.produced.transform is not None:
# An attention's transform, run once its sum is complete.
@@ -216,41 +233,20 @@ def bind_entry(
),
residual_joins_sum=False,
)
capabilities = edge.arriving_plain_add
if not capabilities:
is_plain_add = edge.arriving_plain_add
if is_plain_add is None:
if edge.produced.update is None:
raise ValueError("an arrival must declare its update capabilities")
capabilities = (edge.produced.update.is_plain_add,)
gather_input = (
attn_tp_gather_input
if attn_tp_gather is None
else partial(attn_tp_gather_input, gather=attn_tp_gather)
)
paths = {
capability: _bind_entry_path(
edge,
is_plain_add=capability,
fusions=fusions,
carried_fusions=carried_fusions,
cp_moves=cp_moves,
enters_stack=enters_stack,
attn_input_adapter=attn_input_adapter,
gather_input=gather_input,
)
for capability in capabilities
}
first = next(iter(paths.values()))
if len(paths) == 1:
return first
if any(path.input_move != first.input_move for path in paths.values()):
raise NotImplementedError("update capabilities require different input moves")
return msgspec.structs.replace(
first,
preserves_residual=None,
prepare=partial(
_dispatch_by_update,
paths={capability: path.prepare for capability, path in paths.items()},
),
raise ValueError("an arrival must declare its update capability")
is_plain_add = edge.produced.update.is_plain_add
return _bind_entry_path(
edge,
is_plain_add=is_plain_add,
fusions=fusions,
carried_fusions=carried_fusions,
cp_moves=cp_moves,
enters_stack=enters_stack,
attn_input_adapter=attn_input_adapter,
attn_tp_gather=attn_tp_gather,
)
@@ -263,7 +259,7 @@ def _bind_entry_path(
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
attn_input_adapter: Optional[Callable] = None,
gather_input: Callable = attn_tp_gather_input,
attn_tp_gather: Optional[Callable] = None,
) -> EntryPath:
"""The consumer's half of ``edge``, chosen from the edge's declarations and
the producer's update: what a value carries for a batch (a sum or a handoff
@@ -277,7 +273,7 @@ def _bind_entry_path(
producer chose for a batch."""
# A fused kernel runs the add and the norm itself.
plain = is_plain_add and edge.need.read.is_plain_norm
step, input_move = _select_entry_step(
entry = _select_entry_step(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
@@ -287,12 +283,12 @@ def _bind_entry_path(
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
gather_input=gather_input,
attn_tp_gather=attn_tp_gather,
arrives_written=edge.arrives_written,
)
completed_step = None
if edge.produced.always_partial:
completed_step, _ = _select_entry_step(
completed_step = _select_entry_step(
msgspec.structs.replace(edge.produced, always_partial=False),
residual=edge.residual,
residual_to=edge.residual_to,
@@ -302,12 +298,12 @@ def _bind_entry_path(
residual_joins_sum=False,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
).step
# A written stream entering the first physical layer must not run enter
# again (e.g. MHC expansion). Both alternatives are bound at construction.
written_step = None
if enters_stack:
written_step, _ = _select_entry_step(
written_step = _select_entry_step(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
@@ -317,28 +313,22 @@ def _bind_entry_path(
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=False,
)
).step
preserves_residual = None
if (
edge.produced.always_partial
and edge.produced.group is SumGroup.ATTN_TP
and isinstance(step, partial)
and step.func is _reduce_update_read
and not step.keywords["gathers_residual"]
and entry.ffn_fusions
):
selected = step.keywords["fusions"]
if selected:
# Only the first reachable candidate can certify ownership; an
# earlier custom candidate could otherwise mutate the residual.
preserves_residual = next(
f.preserves_residual for f in fusions if f.run == selected[0]
)
# Only the first reachable candidate can certify ownership; an
# earlier custom candidate could otherwise mutate the residual.
preserves_residual = entry.ffn_fusions[0].preserves_residual
declared_sum = edge.produced.group if edge.produced.always_partial else None
capture_move, capture_move_allocates = _capture_move(edge)
return EntryPath(
prepare=partial(
_run_entry,
step=step,
step=entry.step,
is_plain_add=is_plain_add,
carried_fusions=carried_fusions if plain else (),
expected_sum=declared_sum,
@@ -346,7 +336,9 @@ def _bind_entry_path(
written_step=written_step,
),
input_rows=input_rows(edge),
input_move=input_move,
input_move=entry.input_move,
input_gather_declared=entry.input_gather_declared,
input_retainable=entry.input_retainable,
attn_input_adapter=attn_input_adapter,
capture_move=capture_move,
capture_move_allocates=capture_move_allocates,
@@ -420,9 +412,9 @@ def _select_entry_step(
residual_joins_sum: bool,
cp_moves: Optional[CpMoves],
enters_stack: bool,
gather_input: Callable = attn_tp_gather_input,
attn_tp_gather: Optional[Callable] = None,
arrives_written: bool = False,
) -> Tuple[Callable, Optional[Callable]]:
) -> EntryStep:
"""The steps into a stage for a value that is complete or owes the sum its
producer always leaves, the fused kernels they try first, and the move
after them: complete that sum, move the residual to the rows it has while
@@ -454,7 +446,7 @@ def _select_entry_step(
local = msgspec.structs.replace(
produced, layout=residual_to, group=None, always_partial=False
)
read_local, input_move = _select_entry_step(
inner = _select_entry_step(
local,
residual=residual,
residual_to=residual_to,
@@ -465,11 +457,15 @@ def _select_entry_step(
cp_moves=cp_moves,
enters_stack=enters_stack,
)
return partial(
_move_before_read,
move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back,
read=read_local,
), input_move
return inner._replace(
step=partial(
_move_before_read,
move=cp_moves.reduce_scatter
if owes is not None
else cp_moves.take_back,
read=inner.step,
)
)
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective; a
@@ -489,9 +485,8 @@ def _select_entry_step(
)
else:
step = _attn_tp_slice_update_read
return (
partial(step, scatters_residual=residual != residual_to, read=read),
None,
return EntryStep(
partial(step, scatters_residual=residual != residual_to, read=read)
)
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}:
if residual == residual_to:
@@ -499,13 +494,12 @@ def _select_entry_step(
# rows around it (MHC on an input-scattered batch).
if gathered or owes not in (None, SumGroup.ATTN_TP):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
return EntryStep(
partial(
_tp_reduce_scatter_update_read_gather,
read=read,
reduces=owes is not None,
),
None,
)
)
# A reduce-scatter completes the TP sum onto each rank's slice, which
# the stage takes: its group is the TP group without attention DP or CP.
@@ -516,20 +510,19 @@ def _select_entry_step(
or residual_to.sharded != {TokenAxis.ATTN_TP}
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
return EntryStep(
partial(
_update_read,
pre_move=tp_reduce_scatter if owes is not None else tp_slice,
enters_stack=enters_stack,
read=read,
),
None,
)
)
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the CP moves gather them.
if cp_moves is None:
raise NotImplementedError(f"{produced=} {need=}")
on_chunk, _ = _select_entry_step(
on_chunk = _select_entry_step(
produced,
residual=residual,
residual_to=residual_to,
@@ -539,8 +532,8 @@ def _select_entry_step(
residual_joins_sum=residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
return (partial(cp_moves.gather, gather=on_chunk), None)
).step
return EntryStep(partial(cp_moves.gather, gather=on_chunk))
if gathered == {TokenAxis.ATTN_TP}:
# A complete input on each rank's slice, gathered over attention TP
# once it is read, unless one of the read's own kernels gathers it.
@@ -550,7 +543,7 @@ def _select_entry_step(
or residual_to != produced.layout
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
return EntryStep(
partial(
_update_read,
pre_move=None,
@@ -562,7 +555,13 @@ def _select_entry_step(
else ()
),
),
gather_input,
input_move=(
attn_tp_gather_input
if attn_tp_gather is None
else partial(attn_tp_gather_input, gather=attn_tp_gather)
),
input_gather_declared=attn_tp_gather is not None,
input_retainable=attn_tp_gather is None,
)
if (
residual_to != produced.layout
@@ -581,31 +580,29 @@ def _select_entry_step(
if not gathered:
if owes is None:
if gathers_residual:
return (
return EntryStep(
partial(
_reduce_update_read,
gathers_residual=True,
fusions=(),
group=None,
read=read,
),
None,
)
)
return (
return EntryStep(
partial(
_update_read,
pre_move=None,
enters_stack=enters_stack,
read=read,
),
None,
)
)
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
if owes is not SumGroup.ATTN_TP or not is_plain_add:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (partial(_tp_sum_with_residual_read, read=read), None)
return EntryStep(partial(_tp_sum_with_residual_read, read=read))
# A sum over TP completes only on rows every TP rank holds: the TP group
# spans attention DP and CP.
if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and (
@@ -614,7 +611,7 @@ def _select_entry_step(
raise NotImplementedError(f"{produced=} {need=}")
fused = tuple(f for f in fusions if f.completes is owes)
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
return (
return EntryStep(
partial(
_reduce_update_read,
gathers_residual=gathers_residual,
@@ -623,7 +620,7 @@ def _select_entry_step(
group=owes,
read=read,
),
None,
ffn_fusions=() if gathers_residual else fused,
)
if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP):
raise NotImplementedError(f"{produced=} {need=}")
@@ -640,16 +637,15 @@ def _select_entry_step(
and is_plain_add
and read.is_plain_norm
):
return (
return EntryStep(
partial(
_dp_gather_sum_read,
gathers_residual=gathers_residual,
places_cp_shards=places_cp_shards,
read=read,
),
None,
)
)
return (
return EntryStep(
partial(
_reduce_update_read_dp_gather,
gathers_residual=gathers_residual,
@@ -657,8 +653,7 @@ def _select_entry_step(
group=owes,
places_cp_shards=places_cp_shards,
read=read,
),
None,
)
)
@@ -687,7 +682,8 @@ def _select_exit_move(
update_attn_tp_gather_output,
update=update,
gather=attn_tp_gather,
)
),
gathers_attn_tp=True,
)
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
@@ -247,6 +247,7 @@ class StagePlan:
output=edges.outgoing.produced,
output_move=out.output_move,
output_move_completes_sum=out.output_move_completes_sum,
output_gathers_attn_tp=out.gathers_attn_tp,
returns_over_dp=out.returns_over_dp,
writes_at_handoff=writes_at_handoff,
)
@@ -140,9 +140,9 @@ class EdgeContract(msgspec.Struct, frozen=True):
residual_to: Residual layout after the boundary.
residual_joins_sum: Whether one rank may add the residual into a partial
before reduction; valid only for an eligible plain-add update.
arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for
arriving contributions. Empty means use produced.update's capability.
The actual update object travels with the residual stream.
arriving_plain_add: ResidualUpdate.is_plain_add of the arriving
contribution, whose update object travels with the residual stream;
None means use produced.update's.
arrives_written: Whether the producer applies its update at its exit,
so the stream arrives written with no residual add pending.
"""
@@ -154,8 +154,8 @@ class EdgeContract(msgspec.Struct, frozen=True):
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
arriving_plain_add: Tuple[bool, ...] = ()
# The capability arriving from another layer, not its update object.
arriving_plain_add: Optional[bool] = None
arrives_written: bool = False
@@ -220,6 +220,11 @@ class EntryPath(msgspec.Struct, frozen=True):
applies the producer update and performs the consumer read.
input_rows: Layout handed to compute after preparation and input_move.
input_move: Optional movement after prepare, before compute takes the input.
input_gather_declared: Whether input_move gathers over attention TP
with the stage's own gather.
input_retainable: Whether a capture may keep input_move's output
without copying: it is storage of its own on every path the move
takes.
attn_input_adapter: Optional callable(input, forward_batch, qkv_latent_func) that
adapts already-placed input for attention.
capture_move: Optional movement of the updated residual back onto the
@@ -243,6 +248,8 @@ class EntryPath(msgspec.Struct, frozen=True):
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
input_gather_declared: bool = False
input_retainable: bool = False
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
attn_input_adapter: Optional[Callable] = None
@@ -264,6 +271,8 @@ class StagePath(msgspec.Struct, frozen=True):
output_move: Fixed output transport, or None when absent or chosen per
batch by the attention-DP exit path.
output_move_completes_sum: Whether that move also reduces the output.
output_gathers_attn_tp: Whether that move gathers the rows back over
attention TP.
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
writes_at_handoff: Whether the exit completes the output and writes it
into the residual, for an FFN that hands off to another pipeline rank.
@@ -274,6 +283,7 @@ class StagePath(msgspec.Struct, frozen=True):
# None means no fixed move. returns_over_dp selects batch-dependent DP transport.
output_move: Optional[Callable]
output_move_completes_sum: bool = False
output_gathers_attn_tp: bool = False
returns_over_dp: bool = False
writes_at_handoff: bool = False
@@ -44,10 +44,6 @@ from sglang.srt.layers.layer_boundary.layout import (
moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.layer_boundary.ops import (
attn_tp_gather_input,
update_attn_tp_gather_output,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import (
@@ -526,7 +522,7 @@ def _connect(producer, consumer, *, residual_from=None):
update=None,
)
residual = rows
capabilities = (True,)
arriving_plain_add = True
else:
decl, during, returned = _resolve_stage(before, variant, following=after)
if during is None:
@@ -550,7 +546,7 @@ def _connect(producer, consumer, *, residual_from=None):
before.kind is StageKind.ATTENTION
and before.reduction is ProducerReduction.ALWAYS_PARTIAL
):
arrived, residual, capabilities = decl.output, during, ()
arrived, residual, arriving_plain_add = decl.output, during, None
else:
owes = (
variant is BatchVariant.INPUT_SCATTERED
@@ -574,7 +570,8 @@ def _connect(producer, consumer, *, residual_from=None):
else False,
update=None,
)
residual, capabilities = returned, (before.update.is_plain_add,)
residual = returned
arriving_plain_add = before.update.is_plain_add
written = before.update.applied_at_exit
if after is None:
continue
@@ -606,7 +603,7 @@ def _connect(producer, consumer, *, residual_from=None):
residual,
during,
residual_joins_sum=joins,
arriving_plain_add=capabilities,
arriving_plain_add=arriving_plain_add,
arrives_written=written,
)
entries[variant] = edge
@@ -638,7 +635,7 @@ def _fork_input(prepared, consumer):
declaration.input,
source.residual_to,
during,
arriving_plain_add=(True,),
arriving_plain_add=True,
)
return StageConnection(prepared.consumer, consumer, {}, entries)
@@ -897,13 +894,13 @@ def _check_declared_gathers(appends, bound, *, remote_producer):
]
for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]):
if consumer.declaration.attn_tp_gather is None or any(
_gathers_input(path) for path in consumer.plan.paths.values()
path.entry.input_gather_declared for path in consumer.plan.paths.values()
):
continue
if producer is None and remote_producer:
continue
if producer is not None and any(
_gathers_output(path) for path in producer.plan.paths.values()
path.output_gathers_attn_tp for path in producer.plan.paths.values()
):
continue
error = ValueError(
@@ -914,14 +911,6 @@ def _check_declared_gathers(appends, bound, *, remote_producer):
raise error
def _gathers_input(path):
return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
def _gathers_output(path):
return getattr(path.output_move, "func", None) is update_attn_tp_gather_output
def _ended(declaration, final_read):
"""The stack's last stage as the final read takes it: an FFN on this
rank's attention-TP slice leaves its output there for a read that reads
@@ -263,21 +263,6 @@ def _attn_input_scattered(
return hidden_states
def _dispatch_by_update(
hidden_states, residual, forward_batch, norm, *, paths, update=PLAIN_ADD, **call
):
if residual is None:
# A missing residual cannot join a partial sum. The non-plain path
# performs init_residual/read without that optimization.
prepare = paths[False]
else:
try:
prepare = paths[update.is_plain_add]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
def _run_entry(
hidden_states: Union[torch.Tensor, UnreducedOutput, DeferredFinalize],
residual: Optional[torch.Tensor],
@@ -31,7 +31,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
StagePath,
)
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output
from sglang.srt.layers.layer_boundary.ops import sum_output
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
@@ -395,11 +395,10 @@ class StageBoundary:
forward_batch=forward_batch,
)
)
# Input gathers allocate fresh DP buffers (see attn_tp_gather_input).
# Without a gather this is the mutable residual, so retention copies.
capture_gathered.capture(
gathered_last_layer_output,
owned=move is attn_tp_gather_input
owned=entry.input_retainable
or (
move is None
and entry.capture_preserves_residual is not None