[Refactor] Record what a bound stage path does instead of reading it back (#43453)
This commit is contained in:
@@ -55,7 +55,6 @@ class AttentionInputs:
|
||||
self.hidden_states_local = hidden_states
|
||||
self.forward_batch = forward_batch
|
||||
self.qkv_latent_func = qkv_latent_func
|
||||
self.hidden_states_ = None
|
||||
self.qkv_latent_ = None
|
||||
# When True, hidden_states_local is already attn_tp-gathered upstream
|
||||
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
|
||||
@@ -72,14 +71,6 @@ class AttentionInputs:
|
||||
self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch)
|
||||
return self.qkv_latent_
|
||||
|
||||
def fetch_hidden_states(self):
|
||||
if self.hidden_states_ is not None:
|
||||
return self.hidden_states_
|
||||
self.hidden_states_ = self.hidden_states_local
|
||||
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
||||
self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch)
|
||||
return self.hidden_states_
|
||||
|
||||
|
||||
class AttnTpContext:
|
||||
def __init__(self):
|
||||
@@ -131,11 +122,6 @@ class AttnTpContext:
|
||||
assert attn_inputs is not None
|
||||
return attn_inputs.fetch_qkv_latent()
|
||||
|
||||
def fetch_hidden_states(self):
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
assert attn_inputs is not None
|
||||
return attn_inputs.fetch_hidden_states()
|
||||
|
||||
def clear_attn_inputs(self) -> None:
|
||||
get_forward().set("attn_inputs", None)
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_tp_reduce_scatter_update_read,
|
||||
_attn_tp_slice_update_read,
|
||||
_dispatch_by_update,
|
||||
_dp_gather_sum_read,
|
||||
_move_before_read,
|
||||
_reduce_update_read,
|
||||
@@ -116,6 +115,26 @@ class ExitMove(NamedTuple):
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
returns_over_dp: bool = False
|
||||
# Whether output_move gathers the rows back over attention TP.
|
||||
gathers_attn_tp: bool = False
|
||||
|
||||
|
||||
class EntryStep(NamedTuple):
|
||||
"""The steps chosen into a stage, with what they do that binding and
|
||||
capture rely on."""
|
||||
|
||||
step: Callable
|
||||
# Moves the input onto the stage's rows after the step, or None.
|
||||
input_move: Optional[Callable] = None
|
||||
# input_move gathers over attention TP with the stage's own gather.
|
||||
input_gather_declared: bool = False
|
||||
# input_move returns storage of its own on every path it takes, which a
|
||||
# capture may keep without copying: the boundary's gather does; a stage's
|
||||
# own gather may return a buffer it reuses.
|
||||
input_retainable: bool = False
|
||||
# The kernels the step tries for the sum over attention TP it completes
|
||||
# without moving the residual.
|
||||
ffn_fusions: Tuple[FfnInputFusion, ...] = ()
|
||||
|
||||
|
||||
def input_rows(edge: EdgeContract) -> Layout:
|
||||
@@ -203,8 +222,6 @@ def bind_entry(
|
||||
|
||||
Returns:
|
||||
An EntryPath whose prepare accepts the actual update from the stream.
|
||||
Multiple update capabilities select among preconstructed paths; a single
|
||||
capability needs no runtime dispatcher.
|
||||
"""
|
||||
if edge.produced.transform is not None:
|
||||
# An attention's transform, run once its sum is complete.
|
||||
@@ -216,41 +233,20 @@ def bind_entry(
|
||||
),
|
||||
residual_joins_sum=False,
|
||||
)
|
||||
capabilities = edge.arriving_plain_add
|
||||
if not capabilities:
|
||||
is_plain_add = edge.arriving_plain_add
|
||||
if is_plain_add is None:
|
||||
if edge.produced.update is None:
|
||||
raise ValueError("an arrival must declare its update capabilities")
|
||||
capabilities = (edge.produced.update.is_plain_add,)
|
||||
gather_input = (
|
||||
attn_tp_gather_input
|
||||
if attn_tp_gather is None
|
||||
else partial(attn_tp_gather_input, gather=attn_tp_gather)
|
||||
)
|
||||
paths = {
|
||||
capability: _bind_entry_path(
|
||||
edge,
|
||||
is_plain_add=capability,
|
||||
fusions=fusions,
|
||||
carried_fusions=carried_fusions,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
gather_input=gather_input,
|
||||
)
|
||||
for capability in capabilities
|
||||
}
|
||||
first = next(iter(paths.values()))
|
||||
if len(paths) == 1:
|
||||
return first
|
||||
if any(path.input_move != first.input_move for path in paths.values()):
|
||||
raise NotImplementedError("update capabilities require different input moves")
|
||||
return msgspec.structs.replace(
|
||||
first,
|
||||
preserves_residual=None,
|
||||
prepare=partial(
|
||||
_dispatch_by_update,
|
||||
paths={capability: path.prepare for capability, path in paths.items()},
|
||||
),
|
||||
raise ValueError("an arrival must declare its update capability")
|
||||
is_plain_add = edge.produced.update.is_plain_add
|
||||
return _bind_entry_path(
|
||||
edge,
|
||||
is_plain_add=is_plain_add,
|
||||
fusions=fusions,
|
||||
carried_fusions=carried_fusions,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
@@ -263,7 +259,7 @@ def _bind_entry_path(
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
attn_input_adapter: Optional[Callable] = None,
|
||||
gather_input: Callable = attn_tp_gather_input,
|
||||
attn_tp_gather: Optional[Callable] = None,
|
||||
) -> EntryPath:
|
||||
"""The consumer's half of ``edge``, chosen from the edge's declarations and
|
||||
the producer's update: what a value carries for a batch (a sum or a handoff
|
||||
@@ -277,7 +273,7 @@ def _bind_entry_path(
|
||||
producer chose for a batch."""
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
plain = is_plain_add and edge.need.read.is_plain_norm
|
||||
step, input_move = _select_entry_step(
|
||||
entry = _select_entry_step(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
@@ -287,12 +283,12 @@ def _bind_entry_path(
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
gather_input=gather_input,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
arrives_written=edge.arrives_written,
|
||||
)
|
||||
completed_step = None
|
||||
if edge.produced.always_partial:
|
||||
completed_step, _ = _select_entry_step(
|
||||
completed_step = _select_entry_step(
|
||||
msgspec.structs.replace(edge.produced, always_partial=False),
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
@@ -302,12 +298,12 @@ def _bind_entry_path(
|
||||
residual_joins_sum=False,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
).step
|
||||
# A written stream entering the first physical layer must not run enter
|
||||
# again (e.g. MHC expansion). Both alternatives are bound at construction.
|
||||
written_step = None
|
||||
if enters_stack:
|
||||
written_step, _ = _select_entry_step(
|
||||
written_step = _select_entry_step(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
@@ -317,28 +313,22 @@ def _bind_entry_path(
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=False,
|
||||
)
|
||||
).step
|
||||
preserves_residual = None
|
||||
if (
|
||||
edge.produced.always_partial
|
||||
and edge.produced.group is SumGroup.ATTN_TP
|
||||
and isinstance(step, partial)
|
||||
and step.func is _reduce_update_read
|
||||
and not step.keywords["gathers_residual"]
|
||||
and entry.ffn_fusions
|
||||
):
|
||||
selected = step.keywords["fusions"]
|
||||
if selected:
|
||||
# Only the first reachable candidate can certify ownership; an
|
||||
# earlier custom candidate could otherwise mutate the residual.
|
||||
preserves_residual = next(
|
||||
f.preserves_residual for f in fusions if f.run == selected[0]
|
||||
)
|
||||
# Only the first reachable candidate can certify ownership; an
|
||||
# earlier custom candidate could otherwise mutate the residual.
|
||||
preserves_residual = entry.ffn_fusions[0].preserves_residual
|
||||
declared_sum = edge.produced.group if edge.produced.always_partial else None
|
||||
capture_move, capture_move_allocates = _capture_move(edge)
|
||||
return EntryPath(
|
||||
prepare=partial(
|
||||
_run_entry,
|
||||
step=step,
|
||||
step=entry.step,
|
||||
is_plain_add=is_plain_add,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
expected_sum=declared_sum,
|
||||
@@ -346,7 +336,9 @@ def _bind_entry_path(
|
||||
written_step=written_step,
|
||||
),
|
||||
input_rows=input_rows(edge),
|
||||
input_move=input_move,
|
||||
input_move=entry.input_move,
|
||||
input_gather_declared=entry.input_gather_declared,
|
||||
input_retainable=entry.input_retainable,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
capture_move=capture_move,
|
||||
capture_move_allocates=capture_move_allocates,
|
||||
@@ -420,9 +412,9 @@ def _select_entry_step(
|
||||
residual_joins_sum: bool,
|
||||
cp_moves: Optional[CpMoves],
|
||||
enters_stack: bool,
|
||||
gather_input: Callable = attn_tp_gather_input,
|
||||
attn_tp_gather: Optional[Callable] = None,
|
||||
arrives_written: bool = False,
|
||||
) -> Tuple[Callable, Optional[Callable]]:
|
||||
) -> EntryStep:
|
||||
"""The steps into a stage for a value that is complete or owes the sum its
|
||||
producer always leaves, the fused kernels they try first, and the move
|
||||
after them: complete that sum, move the residual to the rows it has while
|
||||
@@ -454,7 +446,7 @@ def _select_entry_step(
|
||||
local = msgspec.structs.replace(
|
||||
produced, layout=residual_to, group=None, always_partial=False
|
||||
)
|
||||
read_local, input_move = _select_entry_step(
|
||||
inner = _select_entry_step(
|
||||
local,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
@@ -465,11 +457,15 @@ def _select_entry_step(
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return partial(
|
||||
_move_before_read,
|
||||
move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back,
|
||||
read=read_local,
|
||||
), input_move
|
||||
return inner._replace(
|
||||
step=partial(
|
||||
_move_before_read,
|
||||
move=cp_moves.reduce_scatter
|
||||
if owes is not None
|
||||
else cp_moves.take_back,
|
||||
read=inner.step,
|
||||
)
|
||||
)
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective; a
|
||||
@@ -489,9 +485,8 @@ def _select_entry_step(
|
||||
)
|
||||
else:
|
||||
step = _attn_tp_slice_update_read
|
||||
return (
|
||||
partial(step, scatters_residual=residual != residual_to, read=read),
|
||||
None,
|
||||
return EntryStep(
|
||||
partial(step, scatters_residual=residual != residual_to, read=read)
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}:
|
||||
if residual == residual_to:
|
||||
@@ -499,13 +494,12 @@ def _select_entry_step(
|
||||
# rows around it (MHC on an input-scattered batch).
|
||||
if gathered or owes not in (None, SumGroup.ATTN_TP):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_tp_reduce_scatter_update_read_gather,
|
||||
read=read,
|
||||
reduces=owes is not None,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which
|
||||
# the stage takes: its group is the TP group without attention DP or CP.
|
||||
@@ -516,20 +510,19 @@ def _select_entry_step(
|
||||
or residual_to.sharded != {TokenAxis.ATTN_TP}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_update_read,
|
||||
pre_move=tp_reduce_scatter if owes is not None else tp_slice,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
on_chunk, _ = _select_entry_step(
|
||||
on_chunk = _select_entry_step(
|
||||
produced,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
@@ -539,8 +532,8 @@ def _select_entry_step(
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return (partial(cp_moves.gather, gather=on_chunk), None)
|
||||
).step
|
||||
return EntryStep(partial(cp_moves.gather, gather=on_chunk))
|
||||
if gathered == {TokenAxis.ATTN_TP}:
|
||||
# A complete input on each rank's slice, gathered over attention TP
|
||||
# once it is read, unless one of the read's own kernels gathers it.
|
||||
@@ -550,7 +543,7 @@ def _select_entry_step(
|
||||
or residual_to != produced.layout
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_update_read,
|
||||
pre_move=None,
|
||||
@@ -562,7 +555,13 @@ def _select_entry_step(
|
||||
else ()
|
||||
),
|
||||
),
|
||||
gather_input,
|
||||
input_move=(
|
||||
attn_tp_gather_input
|
||||
if attn_tp_gather is None
|
||||
else partial(attn_tp_gather_input, gather=attn_tp_gather)
|
||||
),
|
||||
input_gather_declared=attn_tp_gather is not None,
|
||||
input_retainable=attn_tp_gather is None,
|
||||
)
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
@@ -581,31 +580,29 @@ def _select_entry_step(
|
||||
if not gathered:
|
||||
if owes is None:
|
||||
if gathers_residual:
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_reduce_update_read,
|
||||
gathers_residual=True,
|
||||
fusions=(),
|
||||
group=None,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_update_read,
|
||||
pre_move=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if owes is not SumGroup.ATTN_TP or not is_plain_add:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (partial(_tp_sum_with_residual_read, read=read), None)
|
||||
return EntryStep(partial(_tp_sum_with_residual_read, read=read))
|
||||
# A sum over TP completes only on rows every TP rank holds: the TP group
|
||||
# spans attention DP and CP.
|
||||
if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and (
|
||||
@@ -614,7 +611,7 @@ def _select_entry_step(
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
fused = tuple(f for f in fusions if f.completes is owes)
|
||||
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_reduce_update_read,
|
||||
gathers_residual=gathers_residual,
|
||||
@@ -623,7 +620,7 @@ def _select_entry_step(
|
||||
group=owes,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
ffn_fusions=() if gathers_residual else fused,
|
||||
)
|
||||
if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
@@ -640,16 +637,15 @@ def _select_entry_step(
|
||||
and is_plain_add
|
||||
and read.is_plain_norm
|
||||
):
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_dp_gather_sum_read,
|
||||
gathers_residual=gathers_residual,
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
return (
|
||||
return EntryStep(
|
||||
partial(
|
||||
_reduce_update_read_dp_gather,
|
||||
gathers_residual=gathers_residual,
|
||||
@@ -657,8 +653,7 @@ def _select_entry_step(
|
||||
group=owes,
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -687,7 +682,8 @@ def _select_exit_move(
|
||||
update_attn_tp_gather_output,
|
||||
update=update,
|
||||
gather=attn_tp_gather,
|
||||
)
|
||||
),
|
||||
gathers_attn_tp=True,
|
||||
)
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
|
||||
@@ -247,6 +247,7 @@ class StagePlan:
|
||||
output=edges.outgoing.produced,
|
||||
output_move=out.output_move,
|
||||
output_move_completes_sum=out.output_move_completes_sum,
|
||||
output_gathers_attn_tp=out.gathers_attn_tp,
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
|
||||
@@ -140,9 +140,9 @@ class EdgeContract(msgspec.Struct, frozen=True):
|
||||
residual_to: Residual layout after the boundary.
|
||||
residual_joins_sum: Whether one rank may add the residual into a partial
|
||||
before reduction; valid only for an eligible plain-add update.
|
||||
arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for
|
||||
arriving contributions. Empty means use produced.update's capability.
|
||||
The actual update object travels with the residual stream.
|
||||
arriving_plain_add: ResidualUpdate.is_plain_add of the arriving
|
||||
contribution, whose update object travels with the residual stream;
|
||||
None means use produced.update's.
|
||||
arrives_written: Whether the producer applies its update at its exit,
|
||||
so the stream arrives written with no residual add pending.
|
||||
"""
|
||||
@@ -154,8 +154,8 @@ class EdgeContract(msgspec.Struct, frozen=True):
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
# Capabilities allowed to arrive from another layer, not its update object.
|
||||
arriving_plain_add: Tuple[bool, ...] = ()
|
||||
# The capability arriving from another layer, not its update object.
|
||||
arriving_plain_add: Optional[bool] = None
|
||||
arrives_written: bool = False
|
||||
|
||||
|
||||
@@ -220,6 +220,11 @@ class EntryPath(msgspec.Struct, frozen=True):
|
||||
applies the producer update and performs the consumer read.
|
||||
input_rows: Layout handed to compute after preparation and input_move.
|
||||
input_move: Optional movement after prepare, before compute takes the input.
|
||||
input_gather_declared: Whether input_move gathers over attention TP
|
||||
with the stage's own gather.
|
||||
input_retainable: Whether a capture may keep input_move's output
|
||||
without copying: it is storage of its own on every path the move
|
||||
takes.
|
||||
attn_input_adapter: Optional callable(input, forward_batch, qkv_latent_func) that
|
||||
adapts already-placed input for attention.
|
||||
capture_move: Optional movement of the updated residual back onto the
|
||||
@@ -243,6 +248,8 @@ class EntryPath(msgspec.Struct, frozen=True):
|
||||
# Moves the input onto the stage's rows after prepare, when prepare does
|
||||
# not: (hidden_states, forward_batch) -> hidden_states.
|
||||
input_move: Optional[Callable] = None
|
||||
input_gather_declared: bool = False
|
||||
input_retainable: bool = False
|
||||
# Hands the stage its input once it is on its rows:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
attn_input_adapter: Optional[Callable] = None
|
||||
@@ -264,6 +271,8 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
output_move: Fixed output transport, or None when absent or chosen per
|
||||
batch by the attention-DP exit path.
|
||||
output_move_completes_sum: Whether that move also reduces the output.
|
||||
output_gathers_attn_tp: Whether that move gathers the rows back over
|
||||
attention TP.
|
||||
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
|
||||
writes_at_handoff: Whether the exit completes the output and writes it
|
||||
into the residual, for an FFN that hands off to another pipeline rank.
|
||||
@@ -274,6 +283,7 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
# None means no fixed move. returns_over_dp selects batch-dependent DP transport.
|
||||
output_move: Optional[Callable]
|
||||
output_move_completes_sum: bool = False
|
||||
output_gathers_attn_tp: bool = False
|
||||
|
||||
returns_over_dp: bool = False
|
||||
writes_at_handoff: bool = False
|
||||
|
||||
@@ -44,10 +44,6 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_tp_gather_input,
|
||||
update_attn_tp_gather_output,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
@@ -526,7 +522,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
update=None,
|
||||
)
|
||||
residual = rows
|
||||
capabilities = (True,)
|
||||
arriving_plain_add = True
|
||||
else:
|
||||
decl, during, returned = _resolve_stage(before, variant, following=after)
|
||||
if during is None:
|
||||
@@ -550,7 +546,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
before.kind is StageKind.ATTENTION
|
||||
and before.reduction is ProducerReduction.ALWAYS_PARTIAL
|
||||
):
|
||||
arrived, residual, capabilities = decl.output, during, ()
|
||||
arrived, residual, arriving_plain_add = decl.output, during, None
|
||||
else:
|
||||
owes = (
|
||||
variant is BatchVariant.INPUT_SCATTERED
|
||||
@@ -574,7 +570,8 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
else False,
|
||||
update=None,
|
||||
)
|
||||
residual, capabilities = returned, (before.update.is_plain_add,)
|
||||
residual = returned
|
||||
arriving_plain_add = before.update.is_plain_add
|
||||
written = before.update.applied_at_exit
|
||||
if after is None:
|
||||
continue
|
||||
@@ -606,7 +603,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
residual,
|
||||
during,
|
||||
residual_joins_sum=joins,
|
||||
arriving_plain_add=capabilities,
|
||||
arriving_plain_add=arriving_plain_add,
|
||||
arrives_written=written,
|
||||
)
|
||||
entries[variant] = edge
|
||||
@@ -638,7 +635,7 @@ def _fork_input(prepared, consumer):
|
||||
declaration.input,
|
||||
source.residual_to,
|
||||
during,
|
||||
arriving_plain_add=(True,),
|
||||
arriving_plain_add=True,
|
||||
)
|
||||
return StageConnection(prepared.consumer, consumer, {}, entries)
|
||||
|
||||
@@ -897,13 +894,13 @@ def _check_declared_gathers(appends, bound, *, remote_producer):
|
||||
]
|
||||
for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]):
|
||||
if consumer.declaration.attn_tp_gather is None or any(
|
||||
_gathers_input(path) for path in consumer.plan.paths.values()
|
||||
path.entry.input_gather_declared for path in consumer.plan.paths.values()
|
||||
):
|
||||
continue
|
||||
if producer is None and remote_producer:
|
||||
continue
|
||||
if producer is not None and any(
|
||||
_gathers_output(path) for path in producer.plan.paths.values()
|
||||
path.output_gathers_attn_tp for path in producer.plan.paths.values()
|
||||
):
|
||||
continue
|
||||
error = ValueError(
|
||||
@@ -914,14 +911,6 @@ def _check_declared_gathers(appends, bound, *, remote_producer):
|
||||
raise error
|
||||
|
||||
|
||||
def _gathers_input(path):
|
||||
return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
|
||||
|
||||
|
||||
def _gathers_output(path):
|
||||
return getattr(path.output_move, "func", None) is update_attn_tp_gather_output
|
||||
|
||||
|
||||
def _ended(declaration, final_read):
|
||||
"""The stack's last stage as the final read takes it: an FFN on this
|
||||
rank's attention-TP slice leaves its output there for a read that reads
|
||||
|
||||
@@ -263,21 +263,6 @@ def _attn_input_scattered(
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _dispatch_by_update(
|
||||
hidden_states, residual, forward_batch, norm, *, paths, update=PLAIN_ADD, **call
|
||||
):
|
||||
if residual is None:
|
||||
# A missing residual cannot join a partial sum. The non-plain path
|
||||
# performs init_residual/read without that optimization.
|
||||
prepare = paths[False]
|
||||
else:
|
||||
try:
|
||||
prepare = paths[update.is_plain_add]
|
||||
except KeyError:
|
||||
raise RuntimeError("producer update has no bound input path") from None
|
||||
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _run_entry(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, DeferredFinalize],
|
||||
residual: Optional[torch.Tensor],
|
||||
|
||||
@@ -31,7 +31,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
StagePath,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output
|
||||
from sglang.srt.layers.layer_boundary.ops import sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
@@ -395,11 +395,10 @@ class StageBoundary:
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
)
|
||||
# Input gathers allocate fresh DP buffers (see attn_tp_gather_input).
|
||||
# Without a gather this is the mutable residual, so retention copies.
|
||||
capture_gathered.capture(
|
||||
gathered_last_layer_output,
|
||||
owned=move is attn_tp_gather_input
|
||||
owned=entry.input_retainable
|
||||
or (
|
||||
move is None
|
||||
and entry.capture_preserves_residual is not None
|
||||
|
||||
Reference in New Issue
Block a user