[Refactor] layer-boundary: Bind residual completion to producer updates (#43605)
This commit is contained in:
@@ -173,6 +173,13 @@ class _TransformedRead:
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@property
|
||||
def bind(self):
|
||||
return self._bind if self.inner.bind is not None else None
|
||||
|
||||
def _bind(self, forward_batch, context):
|
||||
return _TransformedRead(self.inner.bind(forward_batch, context), self.transform)
|
||||
|
||||
@property
|
||||
def reads_before_dp_gather(self):
|
||||
return self.inner.reads_before_dp_gather
|
||||
@@ -341,6 +348,14 @@ def _bind_entry_path(
|
||||
step=entry.step,
|
||||
is_plain_add=is_plain_add,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
carried_read_fusions=tuple(
|
||||
f
|
||||
for f in edge.need.read.completing_fusions
|
||||
if f.carried
|
||||
and not f.scatters
|
||||
and (is_plain_add or f.accepts_update is not None)
|
||||
),
|
||||
read=edge.need.read,
|
||||
expected_sum=declared_sum,
|
||||
completed_step=completed_step,
|
||||
written_step=written_step,
|
||||
@@ -367,8 +382,8 @@ def bind_exit(
|
||||
|
||||
Args:
|
||||
edge: Output contract and destination/residual rows. Deferred updates
|
||||
must outlive the producer; pipeline handoffs require a plain add
|
||||
or a residual already written by the producer.
|
||||
must outlive the producer. Pipeline handoff eligibility is checked
|
||||
when the stage is constructed.
|
||||
cp_moves: Context-parallel return operations, when the edge needs them.
|
||||
attn_tp_gather: The stage's implementation of a gather over attention
|
||||
TP, tried before the default one.
|
||||
@@ -383,10 +398,6 @@ def bind_exit(
|
||||
raise NotImplementedError(
|
||||
"a deferred update must guarantee its lifetime across layers"
|
||||
)
|
||||
if not update.is_plain_add and get_parallel().pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"pipeline boundaries require a plain add or a producer-written residual"
|
||||
)
|
||||
if edge.need.layout != edge.residual_to:
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
return _select_exit_move(
|
||||
@@ -398,17 +409,20 @@ def bind_exit(
|
||||
)
|
||||
|
||||
|
||||
def _read_fusions(read, owes, is_plain_add, *, scatters):
|
||||
"""The kernels a read supplies that complete ``owes`` on the rows this
|
||||
entry completes it on. They take the residual add themselves, so only a
|
||||
plain one."""
|
||||
if not is_plain_add:
|
||||
return ()
|
||||
return tuple(
|
||||
f
|
||||
for f in read.completing_fusions
|
||||
if f.completes is owes and f.scatters == scatters
|
||||
def _read_fusion_args(read, owes, is_plain_add, *, scatters):
|
||||
"""Select callback slots for this group and row placement once. Stateful
|
||||
readers resolve those slots on the invocation, stateless readers directly
|
||||
retain their kernels. Non-plain updates require explicit acceptance."""
|
||||
indices = tuple(
|
||||
i
|
||||
for i, f in enumerate(read.completing_fusions)
|
||||
if (is_plain_add or f.accepts_update is not None)
|
||||
and f.completes is owes
|
||||
and f.scatters == scatters
|
||||
)
|
||||
if read.bind is not None:
|
||||
return {"read_fusion_indices": indices}
|
||||
return {"read_fusions": tuple(read.completing_fusions[i] for i in indices)}
|
||||
|
||||
|
||||
def _select_entry_step(
|
||||
@@ -484,7 +498,7 @@ def _select_entry_step(
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return inner._replace(
|
||||
step=partial(_move_before_read, move=move, read=inner.step)
|
||||
step=partial(_move_before_read, move=move, read_step=inner.step)
|
||||
)
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
@@ -501,7 +515,7 @@ def _select_entry_step(
|
||||
if owes is SumGroup.ATTN_TP:
|
||||
step = partial(
|
||||
_attn_tp_reduce_scatter_update_read,
|
||||
read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True),
|
||||
**_read_fusion_args(read, owes, is_plain_add, scatters=True),
|
||||
)
|
||||
else:
|
||||
step = _attn_tp_slice_update_read
|
||||
@@ -581,7 +595,14 @@ def _select_entry_step(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
read_gathers=(
|
||||
read.gathering_reads if is_plain_add or arrives_written else ()
|
||||
read.gathering_reads
|
||||
if read.bind is None and (is_plain_add or arrives_written)
|
||||
else ()
|
||||
),
|
||||
read_gather_indices=(
|
||||
tuple(range(len(read.gathering_reads)))
|
||||
if read.bind is not None and (is_plain_add or arrives_written)
|
||||
else ()
|
||||
),
|
||||
),
|
||||
input_move=(
|
||||
@@ -639,13 +660,13 @@ def _select_entry_step(
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
fused = tuple(f for f in fusions if f.completes is owes)
|
||||
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
|
||||
read_fused = _read_fusion_args(read, owes, is_plain_add, scatters=False)
|
||||
return EntryStep(
|
||||
partial(
|
||||
_reduce_update_read,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
read_fusions=read_fused,
|
||||
**read_fused,
|
||||
group=owes,
|
||||
read=read,
|
||||
),
|
||||
|
||||
@@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_input_default,
|
||||
_attn_input_scattered,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import takes_pending_update
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
@@ -269,6 +270,7 @@ def _bind_stage(
|
||||
*,
|
||||
final_read=None,
|
||||
capture_preserves_residual=None,
|
||||
pipeline_handoff=False,
|
||||
**options,
|
||||
):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
@@ -276,7 +278,15 @@ def _bind_stage(
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
update = declaration.update
|
||||
if declaration.terminal and not (update.is_plain_add or update.applied_at_exit):
|
||||
if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit):
|
||||
raise NotImplementedError(
|
||||
"pipeline boundaries require a producer-written residual"
|
||||
)
|
||||
if declaration.terminal and not (
|
||||
update.is_plain_add
|
||||
or update.applied_at_exit
|
||||
or takes_pending_update(final_read, update)
|
||||
):
|
||||
# The final read adds the last output into the residual itself, as a
|
||||
# plain add, before its norm.
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -186,13 +186,14 @@ class FfnInputFusion(msgspec.Struct, frozen=True):
|
||||
|
||||
class ReadoutFusion(msgspec.Struct, frozen=True):
|
||||
"""A kernel a stage's read supplies (its ``completing_fusions``) that
|
||||
completes the sum its input owes together with the residual add, for a
|
||||
read that is not the residual's plain norm and so takes no FfnInputFusion.
|
||||
completes the input's sum and producer update before the consumer read.
|
||||
|
||||
With no ``accepts_update`` predicate the update must be plain addition.
|
||||
Otherwise ``run`` also receives the actual producer's ``update`` keyword.
|
||||
``run(hidden_states, residual, forward_batch)`` returns the written stream,
|
||||
the completed sum plus the residual, which the read then reads; or None
|
||||
when it does not take the batch, before touching its inputs or starting a
|
||||
collective."""
|
||||
or, with ``reads=True``, takes ``norm`` and returns (input, residual).
|
||||
A carried candidate may receive UnreducedOutput or DeferredFinalize.
|
||||
Returning None must precede any collective or mutation of input/state."""
|
||||
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
@@ -204,6 +205,12 @@ class ReadoutFusion(msgspec.Struct, frozen=True):
|
||||
# Whether it also does the read: run(hidden_states, residual,
|
||||
# forward_batch, norm) then returns the read's (input, residual).
|
||||
reads: bool = False
|
||||
# None restricts the existing kernel to a plain addition. An explicit
|
||||
# predicate validates the actual update before run receives it as a keyword.
|
||||
accepts_update: Optional[Callable] = None
|
||||
# Also accepts dynamically carried output; run must decline unsupported
|
||||
# completion handles without starting communication or changing state.
|
||||
carried: bool = False
|
||||
|
||||
|
||||
class CpMoves(msgspec.Struct, frozen=True):
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
UnreducedOutput,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import bind_update
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
aiter_ar_fusion_applies,
|
||||
flashinfer_ar_fusion_applies,
|
||||
@@ -116,7 +117,7 @@ def exit_facts(kind, plan, variant, path) -> ExitFacts:
|
||||
)
|
||||
may_defer_sum = (
|
||||
tp
|
||||
and not plan.terminal
|
||||
and (not plan.terminal or output.may_defer_to_next)
|
||||
and _sum_deferral_allowed(plan, output, scattered=scattered)
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
@@ -199,10 +200,12 @@ class ExitPolicy:
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
already_reduced: bool = False,
|
||||
update_context=None,
|
||||
):
|
||||
"""Complete the output of the operation-scheduled producer path, which
|
||||
never defers: the sum it owes, then its move and write-back."""
|
||||
steps = self.plan.path_for(forward_batch)
|
||||
update = bind_update(steps.output.update, forward_batch, update_context)
|
||||
if already_reduced and steps.output_move_completes_sum:
|
||||
raise ValueError("already-reduced output cannot use a reduce-scatter move")
|
||||
hidden_states, residual = self._complete_now(
|
||||
@@ -212,6 +215,7 @@ class ExitPolicy:
|
||||
dp_step=None,
|
||||
steps=steps,
|
||||
output_move=steps.output_move,
|
||||
update=update,
|
||||
owes_sum=not already_reduced
|
||||
and steps.output.group is not None
|
||||
and not steps.output_move_completes_sum,
|
||||
@@ -220,7 +224,7 @@ class ExitPolicy:
|
||||
hidden_states,
|
||||
residual,
|
||||
stream,
|
||||
steps.output.update,
|
||||
update,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -260,7 +264,9 @@ class ExitPolicy:
|
||||
group on a sparse layer, the TP group a dense MLP reduces over."""
|
||||
return _sum_group(steps.output.group)
|
||||
|
||||
def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision:
|
||||
def _decide(
|
||||
self, forward_batch: ForwardBatch, steps, *, update=None
|
||||
) -> ExitDecision:
|
||||
"""Decide once, before the FFN runs, what completes its output's sum:
|
||||
the next layer's input, or this layer's postprocess step."""
|
||||
dp_step = self._dp_reduce_scatter_step(forward_batch, steps)
|
||||
@@ -271,6 +277,7 @@ class ExitPolicy:
|
||||
dp_step=dp_step,
|
||||
steps=steps,
|
||||
output_move=steps.output_move,
|
||||
update=update,
|
||||
# A reduce-scatter completes the sum on its way back.
|
||||
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
|
||||
)
|
||||
@@ -340,6 +347,7 @@ class ExitPolicy:
|
||||
steps,
|
||||
output_move: Optional[Callable],
|
||||
owes_sum: bool,
|
||||
update=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the sum the output owes, the move back to where the next layer
|
||||
@@ -364,27 +372,40 @@ class ExitPolicy:
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
update = steps.output.update
|
||||
if update is None:
|
||||
update = bind_update(steps.output.update, forward_batch)
|
||||
if residual is not None and (update.applied_at_exit or steps.writes_at_handoff):
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def mixer_exit(
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
update_context=None,
|
||||
) -> MixerExit:
|
||||
"""Decide once whether this stage's mixer (an attention-like stage)
|
||||
output's sum is completed here or carried on. Call ``finish`` on the
|
||||
result with the mixer's output."""
|
||||
return MixerExit(self, forward_batch, stream=stream)
|
||||
return MixerExit(
|
||||
self, forward_batch, stream=stream, update_context=update_context
|
||||
)
|
||||
|
||||
def ffn_exit(
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
update_context=None,
|
||||
) -> FfnExit:
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call when the FFN may hand
|
||||
off its MoE finalize, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch, stream=stream)
|
||||
return FfnExit(
|
||||
self, forward_batch, stream=stream, update_context=update_context
|
||||
)
|
||||
|
||||
def _defers_sum(
|
||||
self,
|
||||
@@ -485,12 +506,13 @@ class MixerExit:
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
update_context=None,
|
||||
):
|
||||
steps = boundary.plan.path_for(forward_batch)
|
||||
produced = steps.output
|
||||
self._stream = stream
|
||||
self._forward_batch = forward_batch
|
||||
self._update = produced.update
|
||||
self._update = bind_update(produced.update, forward_batch, update_context)
|
||||
self._group = produced.group
|
||||
self._declared_sum = produced.group if produced.always_partial else None
|
||||
self._defers = steps.exit.defers_mixer_sum
|
||||
@@ -552,16 +574,17 @@ class FfnExit:
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
update_context=None,
|
||||
):
|
||||
self._stream = stream
|
||||
self.boundary = boundary
|
||||
# The same path the entry took: the flags that select it hold for the
|
||||
# whole call, and a stage keeps no state between its entry and exit.
|
||||
steps = boundary.plan.path_for(forward_batch)
|
||||
completion = boundary._decide(forward_batch, steps)
|
||||
self._update = bind_update(steps.output.update, forward_batch, update_context)
|
||||
completion = boundary._decide(forward_batch, steps, update=self._update)
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self._complete = completion.complete
|
||||
self._update = steps.output.update
|
||||
# A reduce-scatter that completes the sum on the way back is the
|
||||
# next input's only on an input-scattered batch.
|
||||
self._declared_sum = steps.exit.sum_left_to_next_input
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
ResidualReadout,
|
||||
ResidualUpdate,
|
||||
takes_pending_update,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READOUT,
|
||||
@@ -137,6 +138,7 @@ def _resolve_ffn(
|
||||
update=PLAIN_ADD,
|
||||
dense_tp_size=None,
|
||||
output_complete=False,
|
||||
terminal_accepts_update=False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size):
|
||||
@@ -244,7 +246,7 @@ def _resolve_ffn(
|
||||
rows,
|
||||
group=group,
|
||||
may_defer_to_next=may_leave
|
||||
and not terminal
|
||||
and (not terminal or terminal_accepts_update)
|
||||
and not update.applied_at_exit
|
||||
and update.outlives_layer
|
||||
and output_transform is None,
|
||||
@@ -372,7 +374,9 @@ def _require(protocol, implementation, facts):
|
||||
|
||||
|
||||
def _check_update(update):
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS)
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",))
|
||||
if update.bind is not None and not callable(update.bind):
|
||||
raise TypeError("ResidualUpdate.bind must be callable or None")
|
||||
name = type(update).__name__
|
||||
if update.writes_stream and (update.is_plain_add or not update.applied_at_exit):
|
||||
raise ValueError(
|
||||
@@ -386,7 +390,9 @@ def _check_update(update):
|
||||
|
||||
|
||||
def _check_read(read):
|
||||
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS)
|
||||
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS + ("bind",))
|
||||
if read.bind is not None and not callable(read.bind):
|
||||
raise TypeError("ResidualReadout.bind must be callable or None")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -576,7 +582,7 @@ def _stack_arrival(variant):
|
||||
)
|
||||
|
||||
|
||||
def _resolve_stage(stage, variant, arrival, following=None):
|
||||
def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
|
||||
"""``stage``'s data flow for ``variant``, given what arrives at its entry
|
||||
and the stage after it (None when it ends the stack or a branch)."""
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
@@ -591,6 +597,8 @@ def _resolve_stage(stage, variant, arrival, following=None):
|
||||
update=stage.update,
|
||||
dense_tp_size=stage.dense_tp_size,
|
||||
output_complete=stage.output_complete,
|
||||
terminal_accepts_update=stage.terminal
|
||||
and takes_pending_update(final_read, stage.update),
|
||||
)
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION or (
|
||||
@@ -745,7 +753,9 @@ def _exit_edge(stage, flow, entry):
|
||||
)
|
||||
|
||||
|
||||
def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
|
||||
def _connect_line(
|
||||
line, origins, *, before=None, after=None, arrivals=None, final_read=None
|
||||
):
|
||||
"""The connections along a chain of stages, each resolved once per batch
|
||||
variant: into ``line[0]``, between each two, and out of ``line[-1]`` to
|
||||
``after`` (a stage on the next rank, or None at the end of the stack or a
|
||||
@@ -754,12 +764,33 @@ def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
|
||||
rank's own entry is taken as the stack input's. An error resolving a stage
|
||||
names the append at ``origins[i]`` that declared ``line[i]``, or the one
|
||||
next to the neighbouring rank's stage."""
|
||||
variants = tuple(_active_variants())
|
||||
# The global dense-TP default may be overridden by every FFN declaration.
|
||||
# Odd row counts need a separate path only when some connected FFN would
|
||||
# otherwise slice them over attention TP. Keep neighbours in this decision:
|
||||
# their row choices still determine what crosses the pipeline boundary.
|
||||
needs_unpadded = any(
|
||||
stage is not None
|
||||
and stage.kind is StageKind.FFN
|
||||
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
|
||||
for stage in (before, *line, after)
|
||||
)
|
||||
variants = tuple(
|
||||
variant
|
||||
for variant in _active_variants()
|
||||
if variant is not BatchVariant.UNPADDED or needs_unpadded
|
||||
)
|
||||
|
||||
def resolve(position, stage, arrivals, following=None):
|
||||
try:
|
||||
return {
|
||||
v: _resolve_stage(stage, v, arrivals[v], following) for v in variants
|
||||
v: _resolve_stage(
|
||||
stage,
|
||||
v,
|
||||
arrivals[v],
|
||||
following,
|
||||
final_read=final_read if stage.terminal else None,
|
||||
)
|
||||
for v in variants
|
||||
}
|
||||
except Exception as error:
|
||||
_note_origin(error, origins[position])
|
||||
@@ -1072,6 +1103,7 @@ def _bind_stack(appends, *, previous, following, final_read=None):
|
||||
[append.origin for append, _ in stages],
|
||||
before=chained[0].previous,
|
||||
after=following,
|
||||
final_read=final_read if following is None else None,
|
||||
)
|
||||
_bind_line(
|
||||
stages,
|
||||
@@ -1145,6 +1177,8 @@ def _bind_line(stages, chained, connections, bound, sources, *, final_read=None)
|
||||
connections[position],
|
||||
connections[position + 1],
|
||||
final_read=final_read if position == last else None,
|
||||
pipeline_handoff=position == last
|
||||
and connections[position + 1].consumer is not None,
|
||||
capture_preserves_residual=(
|
||||
_preserves_residual(boundaries[position + 1])
|
||||
if consumer is not None
|
||||
|
||||
@@ -28,6 +28,8 @@ import msgspec
|
||||
class UpdateFacts(msgspec.Struct, frozen=True):
|
||||
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add: bool
|
||||
applied_at_exit: bool
|
||||
outlives_layer: bool
|
||||
@@ -52,6 +54,8 @@ class ReadFacts(msgspec.Struct, frozen=True):
|
||||
Binding takes kernels only from the reads of the stages it binds, never
|
||||
from a stage recorded by its facts, so these supply none."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm: bool
|
||||
reads_before_dp_gather: bool
|
||||
reads_after_attn_tp_gather: bool
|
||||
|
||||
@@ -72,6 +72,12 @@ class DeferredFinalize(msgspec.Struct, frozen=True):
|
||||
there may do that work together with its own; anything else passes it
|
||||
through complete_owed(), which calls ``complete()``."""
|
||||
|
||||
@property
|
||||
def completion_group(self):
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
|
||||
return SumGroup.MOE_OUTPUT
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
"""Do the owed work, unfused, and return the complete output."""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -51,7 +51,11 @@ from sglang.srt.layers.layer_boundary.output import (
|
||||
UnreducedOutput,
|
||||
complete_owed,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
ResidualReadout,
|
||||
ResidualUpdate,
|
||||
bind_read,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -77,19 +81,38 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
|
||||
|
||||
def _run_read_fusions(
|
||||
read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call
|
||||
read_fusions,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
*,
|
||||
read,
|
||||
read_fusion_indices: Tuple[int, ...] = (),
|
||||
update=PLAIN_ADD,
|
||||
**call,
|
||||
):
|
||||
"""The first of the read's own kernels (ReadoutFusion) that takes the
|
||||
batch: one that completes the sum with the residual add hands the written
|
||||
stream to ``read``; one that also reads returns the read. None when every
|
||||
kernel declines."""
|
||||
if read_fusion_indices:
|
||||
read_fusions = (read.completing_fusions[i] for i in read_fusion_indices)
|
||||
for fused in read_fusions:
|
||||
args = {}
|
||||
if fused.accepts_update is None:
|
||||
if not update.is_plain_add:
|
||||
continue
|
||||
else:
|
||||
if not fused.accepts_update(update):
|
||||
continue
|
||||
args["update"] = update
|
||||
if fused.reads:
|
||||
result = fused.run(hidden_states, residual, forward_batch, norm)
|
||||
result = fused.run(hidden_states, residual, forward_batch, norm, **args)
|
||||
if result is not None:
|
||||
return result
|
||||
continue
|
||||
stream = fused.run(hidden_states, residual, forward_batch)
|
||||
stream = fused.run(hidden_states, residual, forward_batch, **args)
|
||||
if stream is not None:
|
||||
return read.read(stream, norm, **call)
|
||||
return None
|
||||
@@ -105,6 +128,7 @@ def _reduce_update_read(
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
read_fusions: Tuple[ReadoutFusion, ...] = (),
|
||||
read_fusion_indices: Tuple[int, ...] = (),
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
@@ -125,6 +149,8 @@ def _reduce_update_read(
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
read_fusion_indices=read_fusion_indices,
|
||||
update=update,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
@@ -271,12 +297,15 @@ def _run_entry(
|
||||
step: Callable,
|
||||
is_plain_add: bool,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
carried_read_fusions: Tuple[ReadoutFusion, ...] = (),
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
expected_sum: Optional[SumGroup] = None,
|
||||
completed_step: Optional[Callable] = None,
|
||||
pending=None,
|
||||
written_step: Optional[Callable] = None,
|
||||
written: bool = False,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
read_context=None,
|
||||
**call,
|
||||
):
|
||||
"""A boundary's half into a stage: complete what the value carries, a sum
|
||||
@@ -287,6 +316,17 @@ def _run_entry(
|
||||
whose declared sum is already completed uses ``completed_step`` instead;
|
||||
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
bind = read.bind
|
||||
if bind is not None:
|
||||
bound = bind_read(read, forward_batch, read_context)
|
||||
declared = read.completing_fusions
|
||||
actual = bound.completing_fusions
|
||||
carried_read_fusions = tuple(
|
||||
actual[i]
|
||||
for i, spec in enumerate(declared)
|
||||
if any(spec is selected for selected in carried_read_fusions)
|
||||
)
|
||||
read = bound
|
||||
if residual is not None and update.is_plain_add != is_plain_add:
|
||||
raise RuntimeError("producer update does not match the boundary's capability")
|
||||
if pending is not None:
|
||||
@@ -299,7 +339,7 @@ def _run_entry(
|
||||
step = completed_step
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
@@ -311,6 +351,30 @@ def _run_entry(
|
||||
if isinstance(owed, UnreducedOutput) and owed.reduce_to_dp_local is not None:
|
||||
hidden_states = complete_owed(owed)
|
||||
else:
|
||||
read_candidates = tuple(
|
||||
fused
|
||||
for fused in carried_read_fusions
|
||||
if (
|
||||
isinstance(owed, DeferredFinalize)
|
||||
and fused.completes is owed.completion_group
|
||||
)
|
||||
or (
|
||||
isinstance(owed, UnreducedOutput)
|
||||
and _sum_group(fused.completes) is owed.group
|
||||
)
|
||||
)
|
||||
result = _run_read_fusions(
|
||||
read_candidates,
|
||||
owed,
|
||||
residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
update=update,
|
||||
**call,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
for fused in carried_fusions:
|
||||
result = fused(
|
||||
owed, residual, forward_batch, call.get("post_residual_addition")
|
||||
@@ -321,6 +385,8 @@ def _run_entry(
|
||||
hidden_states = complete_owed(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
if bind is not None:
|
||||
call["read"] = read
|
||||
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
@@ -335,6 +401,7 @@ def _update_read(
|
||||
enters_stack: bool,
|
||||
read: ResidualReadout,
|
||||
read_gathers: Tuple[Callable, ...] = (),
|
||||
read_gather_indices: Tuple[int, ...] = (),
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
@@ -350,6 +417,8 @@ def _update_read(
|
||||
hidden_states, residual = pre_move(hidden_states, residual)
|
||||
if enters:
|
||||
hidden_states, residual = read.init_residual(hidden_states), None
|
||||
if read_gather_indices:
|
||||
read_gathers = (read.gathering_reads[i] for i in read_gather_indices)
|
||||
for fused in read_gathers:
|
||||
result = fused(hidden_states, residual, norm)
|
||||
if result is not None:
|
||||
@@ -377,6 +446,7 @@ def _attn_tp_reduce_scatter_update_read(
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read_fusions: Tuple[ReadoutFusion, ...] = (),
|
||||
read_fusion_indices: Tuple[int, ...] = (),
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
):
|
||||
@@ -385,7 +455,14 @@ def _attn_tp_reduce_scatter_update_read(
|
||||
of the read's ``read_fusions`` that takes the batch does all of this
|
||||
instead."""
|
||||
result = _run_read_fusions(
|
||||
read_fusions, hidden_states, residual, forward_batch, norm, read=read
|
||||
read_fusions,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
read_fusion_indices=read_fusion_indices,
|
||||
update=update,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
@@ -474,11 +551,11 @@ def _then_attn_cp_gather(
|
||||
|
||||
|
||||
def _move_before_read(
|
||||
hidden_states, residual, forward_batch, norm, *, move, read, **call
|
||||
hidden_states, residual, forward_batch, norm, *, move, read_step, **call
|
||||
):
|
||||
"""Complete and place a producer contribution before its residual update."""
|
||||
hidden_states, residual = move(hidden_states, residual, forward_batch)
|
||||
return read(hidden_states, residual, forward_batch, norm, **call)
|
||||
return read_step(hidden_states, residual, forward_batch, norm, **call)
|
||||
|
||||
|
||||
def _then_moe_cp_gather(
|
||||
@@ -490,6 +567,7 @@ def _then_moe_cp_gather(
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
**call,
|
||||
):
|
||||
"""Read each shard, then gather the MoE group's tokens on its CP path.
|
||||
|
||||
@@ -498,7 +576,7 @@ def _then_moe_cp_gather(
|
||||
ordinary path. The residual stays on this rank's attention rows.
|
||||
"""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call
|
||||
)
|
||||
hidden_states = moe_cp_gather(
|
||||
hidden_states,
|
||||
|
||||
@@ -16,7 +16,7 @@ stage reads its input back from it (its read). The producer of a boundary
|
||||
declares the update, the consumer the read; the boundary steps complete sums
|
||||
and move tokens around the two."""
|
||||
|
||||
from typing import NamedTuple, Optional, Protocol, Tuple
|
||||
from typing import Callable, NamedTuple, Optional, Protocol, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -39,10 +39,18 @@ class ResidualUpdate(Protocol):
|
||||
This is a numerical policy, separate from being a plain add; full
|
||||
precision is always correct.
|
||||
|
||||
An optional ``bind(forward_batch, context)`` returns an update owning this call's
|
||||
dynamic metadata. The stage binds it once before recording a contribution;
|
||||
every capability must match the declaration. Consumption releases the
|
||||
invocation update with its contribution.
|
||||
|
||||
The shard conversion methods must move any update-associated state together
|
||||
with the residual. Nonlinear updates must not claim is_plain_add.
|
||||
"""
|
||||
|
||||
# None explicitly declares a stateless update. Bindings take (batch, context).
|
||||
bind: Optional[Callable] = None
|
||||
|
||||
# A plain add, which one rank may run before the sum it adds into
|
||||
# completes (the DP partial order, a residual that joins the attention
|
||||
# output's sum), and which a fused add + norm kernel may run.
|
||||
@@ -108,6 +116,11 @@ class ResidualReadout(Protocol):
|
||||
snapshot.
|
||||
"""
|
||||
|
||||
# bind(batch, context) may replace callback implementations, preserving the
|
||||
# order and capabilities of completing_fusions and the gathering_reads slots.
|
||||
# None explicitly declares a stateless consumer.
|
||||
bind: Optional[Callable] = None
|
||||
|
||||
# The input is the residual's norm, in the quantization the call asks for,
|
||||
# and the residual is left as it is: what a fused add + norm kernel computes,
|
||||
# and what may run on the rows a sum completes onto.
|
||||
@@ -175,6 +188,73 @@ class ResidualReadout(Protocol):
|
||||
"""
|
||||
|
||||
|
||||
def bind_read(read, forward_batch, context=None):
|
||||
"""Bind a consumer invocation without changing its declared read capabilities."""
|
||||
if read.bind is None:
|
||||
return read
|
||||
bound = read.bind(forward_batch, context)
|
||||
for fact in (
|
||||
"is_plain_norm",
|
||||
"reads_before_dp_gather",
|
||||
"reads_after_attn_tp_gather",
|
||||
):
|
||||
if getattr(bound, fact) != getattr(read, fact):
|
||||
raise ValueError("invocation read changed its declared capability")
|
||||
# Callback slots are selected when the edge is constructed. Binding
|
||||
# replaces their implementations, not their group or row capabilities.
|
||||
declared = read.completing_fusions
|
||||
actual = bound.completing_fusions
|
||||
if len(actual) != len(declared) or len(bound.gathering_reads) != len(
|
||||
read.gathering_reads
|
||||
):
|
||||
raise ValueError("invocation read changed its declared callbacks")
|
||||
for spec, fused in zip(declared, actual):
|
||||
if (spec.completes, spec.scatters, spec.reads, spec.carried) != (
|
||||
fused.completes,
|
||||
fused.scatters,
|
||||
fused.reads,
|
||||
fused.carried,
|
||||
) or (spec.accepts_update is None) != (fused.accepts_update is None):
|
||||
raise ValueError("invocation read changed its declared callbacks")
|
||||
return bound
|
||||
|
||||
|
||||
def bind_update(update, forward_batch, context=None):
|
||||
"""Bind optional per-invocation producer state before recording output.
|
||||
|
||||
``bind(batch, context)`` must preserve every declared update capability and return
|
||||
an update owning its invocation context with ``bind = None``. Stateless updates
|
||||
are unchanged.
|
||||
"""
|
||||
if update.bind is None:
|
||||
return update
|
||||
actual = update.bind(forward_batch, context)
|
||||
for fact in (
|
||||
"is_plain_add",
|
||||
"applied_at_exit",
|
||||
"outlives_layer",
|
||||
"writes_stream",
|
||||
"quantized_sum",
|
||||
):
|
||||
if getattr(actual, fact) != getattr(update, fact):
|
||||
raise ValueError("invocation update changed a declared capability")
|
||||
if actual.bind is not None:
|
||||
raise ValueError("invocation update must have bind = None")
|
||||
return actual
|
||||
|
||||
|
||||
def takes_pending_update(read, update):
|
||||
"""Whether a final read explicitly accepts this producer's update."""
|
||||
return isinstance(read, PendingUpdateRead) and read.accepts_pending_update(update)
|
||||
|
||||
|
||||
class PendingUpdateRead:
|
||||
"""Explicit consumer interface for a terminal pending producer update."""
|
||||
|
||||
def accepts_pending_update(self, update: ResidualUpdate) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FinalRead(Protocol):
|
||||
"""The model's read of the layer stack's output, its final norm.
|
||||
|
||||
@@ -182,6 +262,12 @@ class FinalRead(Protocol):
|
||||
with the output and the residual, returning ``(normed, residual)``: the
|
||||
residual add followed by the norm, so a pending update must be a plain add.
|
||||
|
||||
A ResidualReadout can instead explicitly supply
|
||||
``accepts_pending_update(update)`` and consume the pending producer update
|
||||
through residual_batch.read_output(output, batch, read, norm). Construction
|
||||
checks the declared update; execution checks the actual invocation. This
|
||||
does not change the ordinary two-input final norm contract.
|
||||
|
||||
Optional, each read where it is used with the default given:
|
||||
attn_tp_gather: Its own gather over attention TP of rows the stack's
|
||||
last stage leaves on this rank's attention-TP slice, returning
|
||||
|
||||
@@ -273,6 +273,8 @@ def _norm_quant_kernel(quant_format: str):
|
||||
class PlainAdd:
|
||||
"""A plain residual: the output is added into it."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = True
|
||||
applied_at_exit = False
|
||||
outlives_layer = True
|
||||
@@ -296,6 +298,8 @@ class ReplaceAtExit:
|
||||
previous residual is dropped. The stream is complete: the producer has its
|
||||
boundary complete each part's sum first (StageBoundary.sum_part)."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = True
|
||||
@@ -330,6 +334,8 @@ class NormQuantReadout:
|
||||
It is not applied by the aiter mxfp4 and fp8-group kernels, after an
|
||||
update other than a plain add, or by ``read``, which has no add."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -370,6 +376,8 @@ class NormReadout:
|
||||
``quant_format`` and a ``post_residual_addition``, except that the latter
|
||||
is not applied after an update other than a plain add."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -417,6 +425,8 @@ class UnfusedNormReadout(NormReadout):
|
||||
two steps: the update rounds to the activation dtype before the norm, as
|
||||
in models that add their residual themselves. No fused kernel takes it."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@@ -210,6 +210,8 @@ class _BankReadout:
|
||||
is not the residual's plain norm and no add+norm kernel may claim it. It
|
||||
reads this rank's own rows, which the bank holds, before any DP gather."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
completing_fusions = ()
|
||||
|
||||
@@ -35,6 +35,16 @@ def stream_of(forward_batch):
|
||||
return stream
|
||||
|
||||
|
||||
def clear(forward_batch):
|
||||
"""Release invocation state on completion or an exception."""
|
||||
stream = forward_batch.residual_stream
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
stream.pending.release()
|
||||
stream.pending = stream.residual = None
|
||||
forward_batch.residual_stream = None
|
||||
|
||||
|
||||
def written_residual(forward_batch):
|
||||
"""The residual the last prepare wrote, for a stage whose compute reads it
|
||||
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
|
||||
@@ -56,6 +66,17 @@ def take_output(hidden_states, forward_batch):
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _plain_output(hidden_states, forward_batch):
|
||||
stream = stream_of(forward_batch)
|
||||
stream.check(hidden_states)
|
||||
if stream.pending is not None and not stream.pending.update.is_plain_add:
|
||||
raise NotImplementedError(
|
||||
"output/residual pair requires a plain residual update; "
|
||||
"apply the producer update before export"
|
||||
)
|
||||
return stream
|
||||
|
||||
|
||||
def final_norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
@@ -86,12 +107,12 @@ def final_norm(
|
||||
Normalized tensor. Add and norm remain together to preserve the kernel's
|
||||
accumulation/rounding order; a snapshot is not used as the norm input.
|
||||
"""
|
||||
hidden_states, residual = stream_of(forward_batch).export(
|
||||
hidden_states, residual = _plain_output(hidden_states, forward_batch).export(
|
||||
hidden_states, takes_handoff=finalize_norm is not None
|
||||
)
|
||||
# The terminal consumer now owns the pair. Do not keep layer buffers alive
|
||||
# through logits processing or the next forward on this batch.
|
||||
forward_batch.residual_stream = None
|
||||
clear(forward_batch)
|
||||
from sglang.srt.layers.layer_boundary.output import DeferredFinalize
|
||||
|
||||
if isinstance(hidden_states, DeferredFinalize):
|
||||
@@ -114,6 +135,83 @@ def final_norm(
|
||||
)
|
||||
|
||||
|
||||
def read_output(hidden_states, forward_batch, read, norm, *, read_context=None):
|
||||
"""Consume the stack's output with an explicitly declared final read.
|
||||
|
||||
This entry accepts producer updates through ``accepts_pending_update``.
|
||||
Completion candidates see the actual update before any collective; their
|
||||
fallback completes the owed output once, then applies update_and_read.
|
||||
A bindable read receives read_context once before consuming the output.
|
||||
Ordinary model final norms continue to use final_norm().
|
||||
"""
|
||||
from sglang.srt.layers.layer_boundary.layout import _sum_group
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
DeferredFinalize,
|
||||
UnreducedOutput,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import _run_read_fusions
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
bind_read,
|
||||
takes_pending_update,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
|
||||
|
||||
stream = stream_of(forward_batch)
|
||||
stream.check(hidden_states)
|
||||
pending = stream.pending
|
||||
try:
|
||||
if pending is not None and not takes_pending_update(read, pending.update):
|
||||
raise NotImplementedError(
|
||||
"final read does not accept the pending producer update"
|
||||
)
|
||||
read = bind_read(read, forward_batch, read_context)
|
||||
if pending is None:
|
||||
value = stream.residual
|
||||
if value is None:
|
||||
value = read.init_residual(hidden_states)
|
||||
return read.read(value, norm)
|
||||
if not takes_pending_update(read, pending.update):
|
||||
raise NotImplementedError(
|
||||
"final read does not accept the pending producer update"
|
||||
)
|
||||
owed = pending.owed
|
||||
candidates = tuple(
|
||||
f
|
||||
for f in read.completing_fusions
|
||||
if not f.scatters
|
||||
and (
|
||||
(isinstance(owed, DeclaredSum) and f.completes is owed.group)
|
||||
or (
|
||||
isinstance(owed, UnreducedOutput)
|
||||
and f.carried
|
||||
and owed.reduce_to_dp_local is None
|
||||
and _sum_group(f.completes) is owed.group
|
||||
)
|
||||
or (
|
||||
isinstance(owed, DeferredFinalize)
|
||||
and f.carried
|
||||
and f.completes is owed.completion_group
|
||||
)
|
||||
)
|
||||
)
|
||||
result = _run_read_fusions(
|
||||
candidates,
|
||||
pending.for_boundary(),
|
||||
stream.residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
update=pending.update,
|
||||
)
|
||||
if result is None:
|
||||
value = pending.complete()
|
||||
result = read.update_and_read(pending.update, value, stream.residual, norm)
|
||||
return result
|
||||
finally:
|
||||
# The terminal consumer owns the result, including on an exception.
|
||||
clear(forward_batch)
|
||||
|
||||
|
||||
def to_pp(hidden_states, forward_batch):
|
||||
"""Export hidden_states and residual as PPProxyTensors.
|
||||
|
||||
@@ -129,8 +227,10 @@ def to_pp(hidden_states, forward_batch):
|
||||
and is sent as hidden_states alone; the receiver's from_pp reconstructs it
|
||||
as written.
|
||||
"""
|
||||
hidden_states, residual = stream_of(forward_batch).export(hidden_states)
|
||||
forward_batch.residual_stream = None
|
||||
hidden_states, residual = _plain_output(hidden_states, forward_batch).export(
|
||||
hidden_states
|
||||
)
|
||||
clear(forward_batch)
|
||||
tensors = {"hidden_states": hidden_states}
|
||||
if residual is not None:
|
||||
tensors["residual"] = residual
|
||||
|
||||
@@ -134,6 +134,8 @@ class _AttnReadout:
|
||||
the previous layer's output: such a layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -167,6 +169,8 @@ class _AttnUpdate:
|
||||
residual its read produced. The injection is not a plain add, so it runs
|
||||
only once the sum it writes in is complete."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
@@ -189,6 +193,8 @@ class _AttnUpdate:
|
||||
class _FfnReadout:
|
||||
"""The attention output's injection and the FFN input's mix."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -220,6 +226,8 @@ class _FfnUpdate:
|
||||
"""The gated injection of the FFN output, which this layer runs itself. The
|
||||
streams stay widened: the layer stack's terminal read mixes them down."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
|
||||
@@ -103,6 +103,8 @@ class _AttnReadout:
|
||||
already hold the previous layer's output: an iHC layer takes its input
|
||||
written back. A ``post_residual_addition`` is not applied."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -129,6 +131,8 @@ class _AttnUpdate:
|
||||
It is not a plain add, so it runs only once the sum it writes in is
|
||||
complete."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
@@ -152,6 +156,8 @@ class _FfnReadout:
|
||||
"""The attention output's write-back and the FFN input's gated mix and
|
||||
norm, fused in post_pre when the layer provides it."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -181,6 +187,8 @@ class _FfnUpdate:
|
||||
layer stack's terminal read is a learned head rather than a contraction,
|
||||
so it belongs to the final norm."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
|
||||
@@ -139,6 +139,8 @@ class _AttnReadout:
|
||||
layer's output: an MHC layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -164,6 +166,8 @@ class _AttnUpdate:
|
||||
"""hc_post with the coefficients the attention's read produced. It is not a
|
||||
plain add, so it runs only once the sum it writes in is complete."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
@@ -187,6 +191,8 @@ class _FfnReadout:
|
||||
"""The attention output's hc_post and the FFN input's hc_pre and norm, fused
|
||||
in hc_ffn_post_pre when it takes the batch."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
@@ -215,6 +221,8 @@ class _FfnUpdate:
|
||||
layer runs itself; the last layer also contracts the streams into the
|
||||
hidden states the layer stack hands on."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
|
||||
@@ -24,6 +24,8 @@ class PostNormAdd:
|
||||
"""``residual + norm(output)``. It is not a plain add, so it runs only on a
|
||||
complete output. ``applied_at_exit`` makes the stage write it itself."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_add = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
@@ -46,6 +48,8 @@ class PostNormAdd:
|
||||
class PlainReadout:
|
||||
"""The input is the residual itself; the stage's norm is not applied."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@@ -46,7 +46,7 @@ class Contribution(msgspec.Struct):
|
||||
"""
|
||||
|
||||
value: Optional[torch.Tensor]
|
||||
update: ResidualUpdate
|
||||
update: Optional[ResidualUpdate]
|
||||
owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None
|
||||
|
||||
def for_boundary(self):
|
||||
@@ -66,9 +66,10 @@ class Contribution(msgspec.Struct):
|
||||
return value
|
||||
|
||||
def release(self):
|
||||
"""Drop the tensors of a contribution its consumer has taken."""
|
||||
"""Drop a consumed contribution and its per-invocation update state."""
|
||||
self.value = None
|
||||
self.owed = None
|
||||
self.update = None
|
||||
|
||||
|
||||
class OwedOutput(msgspec.Struct, frozen=True):
|
||||
@@ -134,7 +135,8 @@ class ResidualStream:
|
||||
Args:
|
||||
output: Complete tensor, UnreducedOutput, or producer-specific DeferredFinalize.
|
||||
update: Producer operation that the next boundary must apply.
|
||||
declared_sum: SumGroup statically owed by a raw tensor, or None. Cannot
|
||||
declared_sum: The producer's outstanding sum contract,
|
||||
statically owed by a raw tensor; None when complete. Cannot
|
||||
be combined with another output wrapper's completion contract.
|
||||
|
||||
Returns:
|
||||
@@ -146,13 +148,23 @@ class ResidualStream:
|
||||
if declared_sum is not None:
|
||||
if not isinstance(output, torch.Tensor):
|
||||
raise RuntimeError("a declared sum cannot carry another completion")
|
||||
contribution = Contribution(output, update, DeclaredSum(declared_sum))
|
||||
owed = (
|
||||
declared_sum
|
||||
if isinstance(declared_sum, DeclaredSum)
|
||||
else DeclaredSum(declared_sum)
|
||||
)
|
||||
contribution = Contribution(output, update, owed)
|
||||
elif isinstance(output, UnreducedOutput):
|
||||
contribution = Contribution(output.partial, update, output)
|
||||
elif isinstance(output, DeferredFinalize):
|
||||
contribution = Contribution(None, update, output)
|
||||
else:
|
||||
contribution = Contribution(output, update)
|
||||
bind = update.bind
|
||||
if bind is not None:
|
||||
raise RuntimeError(
|
||||
"bind an invocation update at the stage before recording it"
|
||||
)
|
||||
self.pending = contribution
|
||||
return OwedOutput(contribution) if contribution.owed is not None else output
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.ops import sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual import bind_update
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
@@ -168,7 +169,7 @@ class StageBoundary:
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
def finish(self, hidden_states, forward_batch, *, update_context=None):
|
||||
"""Hand this stage's output to the following stage: an attention's
|
||||
actual contribution, or an FFN's or a mixer's output with its sum
|
||||
completed or carried on. An FFN whose boundary has fusions (which may
|
||||
@@ -177,16 +178,18 @@ class StageBoundary:
|
||||
produced = self.plan.path_for(forward_batch).output
|
||||
return stream_of(forward_batch).record(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
bind_update(produced.update, forward_batch, update_context),
|
||||
declared_sum=produced.group if produced.always_partial else None,
|
||||
)
|
||||
if self.kind is StageKind.FFN and self.plan.fusions is not None:
|
||||
raise RuntimeError(
|
||||
"an FFN whose boundary has fusions finishes through exit()"
|
||||
)
|
||||
return self.exit(forward_batch).finish(hidden_states)
|
||||
return self.exit(forward_batch, update_context=update_context).finish(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
def exit(self, forward_batch, *, update_context=None):
|
||||
"""Select and scope one FFN or single-stage mixer's output decision,
|
||||
for an FFN whose MoE may hand off its finalize (see finish()).
|
||||
|
||||
@@ -201,10 +204,16 @@ class StageBoundary:
|
||||
using the same decision rather than selecting a second path.
|
||||
"""
|
||||
result = (
|
||||
self.plan.output.ffn_exit(forward_batch, stream=stream_of(forward_batch))
|
||||
self.plan.output.ffn_exit(
|
||||
forward_batch,
|
||||
stream=stream_of(forward_batch),
|
||||
update_context=update_context,
|
||||
)
|
||||
if self.kind is StageKind.FFN
|
||||
else self.plan.output.mixer_exit(
|
||||
forward_batch, stream=stream_of(forward_batch)
|
||||
forward_batch,
|
||||
stream=stream_of(forward_batch),
|
||||
update_context=update_context,
|
||||
)
|
||||
)
|
||||
return result
|
||||
@@ -260,7 +269,14 @@ class StageBoundary:
|
||||
hidden_states = stream.complete(hidden_states)
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def complete_now(self, hidden_states, forward_batch, *, already_reduced=False):
|
||||
def complete_now(
|
||||
self,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
*,
|
||||
already_reduced=False,
|
||||
update_context=None,
|
||||
):
|
||||
"""Complete an FFN output outside exit(), including its residual update.
|
||||
|
||||
already_reduced is for a producer kernel that fused the output sum.
|
||||
@@ -271,6 +287,7 @@ class StageBoundary:
|
||||
stream_of(forward_batch),
|
||||
forward_batch,
|
||||
already_reduced=already_reduced,
|
||||
update_context=update_context,
|
||||
)
|
||||
|
||||
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
|
||||
@@ -347,6 +364,7 @@ class StageBoundary:
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
quant_format: str = "",
|
||||
capture: Optional[Callable] = None,
|
||||
read_context=None,
|
||||
):
|
||||
# Aux consumers need a materialized output before the input norm.
|
||||
# Complete communication here, preserving the existing add+norm kernel
|
||||
@@ -373,6 +391,7 @@ class StageBoundary:
|
||||
steps,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
read_context=read_context,
|
||||
)
|
||||
entry = steps.entry
|
||||
if capture is not None and not capture_before_read:
|
||||
|
||||
@@ -133,6 +133,8 @@ class _ResidualMergeRead:
|
||||
sum as the new residual, and the input norm of that sum in the model
|
||||
dtype. The producer's update must be a plain add."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Small fixtures around production boundaries; no parallel layout recipes."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.layer_boundary import factories
|
||||
from sglang.srt.layers.layer_boundary.construction import StagePlan
|
||||
from sglang.srt.layers.layer_boundary.factories import (
|
||||
@@ -14,6 +17,8 @@ from sglang.srt.layers.layer_boundary.factories import (
|
||||
from sglang.srt.layers.layer_boundary.ops import keep_output
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL_OPS
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import utils as moe_utils
|
||||
from sglang.test.communicator_patch import patch_communicator
|
||||
|
||||
|
||||
def build_stages(*stages, previous=None, terminal=False):
|
||||
@@ -189,3 +194,130 @@ def postprocess_output(boundary, hidden, residual, forward_batch):
|
||||
|
||||
def identity_input(hidden_states, forward_batch):
|
||||
return hidden_states
|
||||
|
||||
|
||||
def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
|
||||
fields = dict(
|
||||
tp_size=attn_dp * attn_cp * attn_tp,
|
||||
tp_rank=0,
|
||||
attn_dp_size=attn_dp,
|
||||
attn_tp_size=attn_tp,
|
||||
attn_tp_rank=0,
|
||||
attn_cp_size=attn_cp,
|
||||
attn_cp_rank=0,
|
||||
moe_dense_tp_size=None,
|
||||
enable_prefill_cp=False,
|
||||
enable_cp_tp_group_sharing=False,
|
||||
moe_ep_size=1,
|
||||
moe_tp_size=attn_dp * attn_cp * attn_tp,
|
||||
moe_dp_size=1,
|
||||
dwdp_size=1,
|
||||
attn_dp_enabled=attn_dp > 1,
|
||||
enable_attn_tp_input_scattered=False,
|
||||
disable_attn_tp_gather=False,
|
||||
tp_group=SimpleNamespace(
|
||||
name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp))
|
||||
),
|
||||
attn_tp_group=SimpleNamespace(name="attn_tp", ranks=list(range(attn_tp))),
|
||||
attn_cp_group=SimpleNamespace(name="attn_cp", ranks=list(range(attn_cp))),
|
||||
)
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def planning(
|
||||
parallel,
|
||||
*,
|
||||
sp=False,
|
||||
a2a=False,
|
||||
dsa_cp=False,
|
||||
boundary_reduction="rs+rsv",
|
||||
):
|
||||
"""What layer planning and communicator construction read, without the
|
||||
process-wide parallel state. ``parallel`` may be a callable, for a
|
||||
per-thread parallel state. ``dsa_cp``: the prefill CP is DSA's (MLA's is
|
||||
the same to the communicator)."""
|
||||
get_parallel = parallel if callable(parallel) else (lambda: parallel)
|
||||
|
||||
def moe_cp_gathers():
|
||||
return get_parallel().attn_cp_size > get_parallel().moe_dp_size
|
||||
|
||||
with (
|
||||
patch_communicator("get_parallel", get_parallel),
|
||||
patch_communicator("is_dsa_enable_prefill_cp", lambda: dsa_cp),
|
||||
patch_communicator("is_mla_cp_enabled", lambda: False),
|
||||
patch_communicator(
|
||||
"get_moe_cp_size",
|
||||
lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1,
|
||||
),
|
||||
patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp),
|
||||
patch_communicator(
|
||||
"get_spec", lambda: SimpleNamespace(speculative_algorithm=None)
|
||||
),
|
||||
patch_communicator(
|
||||
"get_moe_a2a_backend",
|
||||
lambda: SimpleNamespace(is_none=lambda: not a2a),
|
||||
),
|
||||
patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a),
|
||||
patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers),
|
||||
patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)),
|
||||
# A MoE whose EP and TP sums merge: its output's group is the TP group.
|
||||
patch_communicator(
|
||||
"post_experts_reduction_group", lambda: get_parallel().tp_group
|
||||
),
|
||||
# Planning asks whether a dense layer gathers for two-batch overlap.
|
||||
patch_communicator(
|
||||
"get_exec",
|
||||
lambda: SimpleNamespace(
|
||||
comm=SimpleNamespace(
|
||||
boundary_reduction=boundary_reduction,
|
||||
enable_quant_communications=False,
|
||||
),
|
||||
overlap=SimpleNamespace(enable_two_batch_overlap=False),
|
||||
),
|
||||
),
|
||||
# What the MoE predicates an exit's decisions read, in their module.
|
||||
patch.object(moe_utils, "get_parallel", get_parallel),
|
||||
patch.object(
|
||||
moe_utils,
|
||||
"get_moe_a2a_backend",
|
||||
lambda: SimpleNamespace(
|
||||
is_none=lambda: not a2a,
|
||||
is_flashinfer=lambda: False,
|
||||
is_pplx=lambda: False,
|
||||
is_flashinfer_megamoe=lambda: False,
|
||||
),
|
||||
),
|
||||
patch.object(moe_utils, "get_lora", lambda: SimpleNamespace(enable_lora=False)),
|
||||
patch.object(
|
||||
moe_utils,
|
||||
"get_exec",
|
||||
lambda: SimpleNamespace(
|
||||
comm=SimpleNamespace(enable_quant_communications=False)
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
moe_utils, "post_experts_reduction_group", lambda: get_parallel().tp_group
|
||||
),
|
||||
patch.object(
|
||||
moe_utils, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False
|
||||
),
|
||||
patch.object(
|
||||
moe_utils,
|
||||
"is_dp_attention_enabled",
|
||||
lambda: get_parallel().attn_dp_size > 1,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
class Norm:
|
||||
"""add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h."""
|
||||
|
||||
def __call__(self, x, residual=None, post_residual_addition=None):
|
||||
assert post_residual_addition is None
|
||||
if residual is None:
|
||||
return x * 2
|
||||
s = x + residual
|
||||
return s * 2, s
|
||||
|
||||
Reference in New Issue
Block a user