[Refactor] Make residual read and update capabilities explicit and checked (#43455)
This commit is contained in:
@@ -166,11 +166,19 @@ class _TransformedRead:
|
||||
transform: OutputTransform
|
||||
|
||||
is_plain_norm = False
|
||||
# The inner read's kernels read the stream directly and would skip the
|
||||
# transform, so none is offered.
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@property
|
||||
def reads_before_dp_gather(self):
|
||||
return self.inner.reads_before_dp_gather
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
return self.inner.reads_after_attn_tp_gather
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.inner.init_residual(hidden_states)
|
||||
|
||||
@@ -368,8 +376,8 @@ def bind_exit(
|
||||
transport. The receiver binds its read independently.
|
||||
"""
|
||||
update = edge.produced.update
|
||||
if not getattr(update, "applied_at_exit", False):
|
||||
if not getattr(update, "outlives_layer", False):
|
||||
if not update.applied_at_exit:
|
||||
if not update.outlives_layer:
|
||||
raise NotImplementedError(
|
||||
"a deferred update must guarantee its lifetime across layers"
|
||||
)
|
||||
@@ -396,7 +404,7 @@ def _read_fusions(read, owes, is_plain_add, *, scatters):
|
||||
return ()
|
||||
return tuple(
|
||||
f
|
||||
for f in getattr(read, "completing_fusions", ())
|
||||
for f in read.completing_fusions
|
||||
if f.completes is owes and f.scatters == scatters
|
||||
)
|
||||
|
||||
@@ -558,9 +566,7 @@ def _select_entry_step(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
read_gathers=(
|
||||
getattr(read, "gathering_reads", ())
|
||||
if is_plain_add or arrives_written
|
||||
else ()
|
||||
read.gathering_reads if is_plain_add or arrives_written else ()
|
||||
),
|
||||
),
|
||||
input_move=(
|
||||
|
||||
@@ -313,6 +313,14 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
update = declaration.update
|
||||
if declaration.terminal and not (update.is_plain_add or update.applied_at_exit):
|
||||
# The final read adds the last output into the residual itself, as a
|
||||
# plain add, before its norm.
|
||||
raise NotImplementedError(
|
||||
f"the layer stack ends on a stage whose {type(update).__name__} "
|
||||
"update the final read would apply as a plain add"
|
||||
)
|
||||
variants = {}
|
||||
for variant, edge in incoming.entries.items():
|
||||
if (
|
||||
@@ -321,13 +329,17 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with a gather over attention CP"
|
||||
)
|
||||
attn_input_adapter = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
# On an input-scattered batch the step gathers the rows itself for
|
||||
# an attention whose QKV hook does not gather them after the
|
||||
# projection.
|
||||
attn_input_adapter = (
|
||||
_attn_input_scattered
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and not declaration.update.is_plain_add
|
||||
else _attn_input_default
|
||||
)
|
||||
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
|
||||
|
||||
@@ -271,7 +271,7 @@ class ExitPolicy:
|
||||
hidden_states,
|
||||
steps.output.group,
|
||||
forward_batch,
|
||||
may_quantize=steps.output.update.is_plain_add,
|
||||
may_quantize=steps.output.update.quantized_sum,
|
||||
)
|
||||
if steps.output.transform is not None:
|
||||
hidden_states = steps.output.transform.apply(hidden_states)
|
||||
@@ -476,7 +476,7 @@ class MixerExit:
|
||||
hidden_states,
|
||||
self._group,
|
||||
self._forward_batch,
|
||||
may_quantize=self._update.is_plain_add,
|
||||
may_quantize=self._update.quantized_sum,
|
||||
)
|
||||
return self._stream.record(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
|
||||
@@ -45,12 +45,14 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
ResidualReadout,
|
||||
ResidualUpdate,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READOUT,
|
||||
NORM_READOUT,
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
@@ -150,6 +152,8 @@ def _resolve_ffn(
|
||||
group = SumGroup.MOE_OUTPUT
|
||||
else:
|
||||
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update.writes_stream or output_complete
|
||||
if variant is BatchVariant.SEQUENCE_PARALLEL:
|
||||
return (
|
||||
StageContract(
|
||||
@@ -172,8 +176,9 @@ def _resolve_ffn(
|
||||
InputContract(full, read=read),
|
||||
OutputContract(
|
||||
attention,
|
||||
group=group,
|
||||
may_reduce_scatter=use_reduce_scatter
|
||||
group=None if complete else group,
|
||||
may_reduce_scatter=not complete
|
||||
and use_reduce_scatter
|
||||
and (scattered_residual or not terminal),
|
||||
update=update,
|
||||
transform=output_transform,
|
||||
@@ -199,8 +204,6 @@ def _resolve_ffn(
|
||||
rows = Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
|
||||
produced = (
|
||||
OutputContract(rows, update=update, transform=output_transform)
|
||||
if complete
|
||||
@@ -301,6 +304,51 @@ class StageDeclaration:
|
||||
for source in (self.previous, self.prepared_from):
|
||||
if source is not None and not isinstance(source, StageDeclaration):
|
||||
raise TypeError("stage sources must be declarations")
|
||||
_check_update(self.update)
|
||||
_check_read(self.read)
|
||||
|
||||
|
||||
# The members every ResidualUpdate and ResidualReadout must set: the binding
|
||||
# reads each of them, and none has a default that is correct for every
|
||||
# implementation.
|
||||
_UPDATE_FACTS = (
|
||||
"is_plain_add",
|
||||
"applied_at_exit",
|
||||
"outlives_layer",
|
||||
"writes_stream",
|
||||
"quantized_sum",
|
||||
)
|
||||
_READ_FACTS = ("is_plain_norm", "reads_before_dp_gather", "reads_after_attn_tp_gather")
|
||||
# The kernels a read supplies, which the binding also reads; empty when it
|
||||
# supplies none.
|
||||
_READ_KERNELS = ("completing_fusions", "gathering_reads")
|
||||
|
||||
|
||||
def _require(protocol, implementation, facts):
|
||||
missing = [name for name in facts if not hasattr(implementation, name)]
|
||||
if missing:
|
||||
raise TypeError(
|
||||
f"{type(implementation).__name__} does not set {', '.join(missing)}, "
|
||||
f"which every {protocol} must"
|
||||
)
|
||||
|
||||
|
||||
def _check_update(update):
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS)
|
||||
name = type(update).__name__
|
||||
if update.writes_stream and (update.is_plain_add or not update.applied_at_exit):
|
||||
raise ValueError(
|
||||
f"{name} writes the next stream itself, which only an update "
|
||||
"applied at its exit and other than a plain add does"
|
||||
)
|
||||
if update.quantized_sum and not update.is_plain_add:
|
||||
raise ValueError(
|
||||
f"{name} lets its sum run quantized, which only a plain add does"
|
||||
)
|
||||
|
||||
|
||||
def _check_read(read):
|
||||
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -420,11 +468,13 @@ def _resolve_stage(stage, variant, following=None):
|
||||
if stage.update.applied_at_exit:
|
||||
if stage.sparse and moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
"an update applied at the stage's exit with a MoE gathered over "
|
||||
"the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_scattered_possible():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
"an update applied at the stage's exit with input-scattered "
|
||||
"attention under attention CP"
|
||||
)
|
||||
if stage.kind is StageKind.FFN:
|
||||
declaration, residual, returned = _resolve_ffn(
|
||||
@@ -439,8 +489,7 @@ def _resolve_stage(stage, variant, following=None):
|
||||
)
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION or (
|
||||
following is not None
|
||||
and getattr(following.read, "reads_after_attn_tp_gather", False)
|
||||
following is not None and following.read.reads_after_attn_tp_gather
|
||||
):
|
||||
returned = attention
|
||||
elif exit_rows is ExitRows.SLICE:
|
||||
@@ -706,11 +755,12 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
stack's first. What they build is discarded.
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
final_read: The model's final read of the stack's output, when it is
|
||||
not a plain final norm: the consumer of the last stage when no
|
||||
later layer declares one. Its ``attn_tp_gather`` gathers the rows
|
||||
that stage leaves on this rank's attention-TP slice, and with
|
||||
``reads_attn_tp_slices`` it reads that slice instead.
|
||||
final_read: The model's final read of the stack's output (a
|
||||
``FinalRead``), when it is not a plain final norm: the consumer of
|
||||
the last stage when no later layer declares one. Its
|
||||
``attn_tp_gather`` gathers the rows that stage leaves on this
|
||||
rank's attention-TP slice, and with ``reads_attn_tp_slices`` it
|
||||
reads that slice instead.
|
||||
"""
|
||||
global _stack
|
||||
outer = _stack
|
||||
|
||||
@@ -135,9 +135,8 @@ def _reduce_update_read(
|
||||
if result is not None:
|
||||
return result
|
||||
if group is SumGroup.ATTN_TP:
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = attn_tp_all_reduce(
|
||||
hidden_states, forward_batch, may_quantize=update.is_plain_add
|
||||
hidden_states, forward_batch, may_quantize=update.quantized_sum
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
@@ -31,6 +31,13 @@ class ResidualUpdate(Protocol):
|
||||
than leaving the operation for the next prepare.
|
||||
outlives_layer: Whether parameters/state remain valid after
|
||||
leaving the producer layer, including any offload or state reuse.
|
||||
writes_stream: Whether the producer computes the next stream itself
|
||||
and writes it at its exit, dropping the previous residual: what
|
||||
reaches its exit is already complete.
|
||||
quantized_sum: Whether the sum the contribution owes may be completed
|
||||
by a quantized all-reduce when quantized communication is enabled.
|
||||
This is a numerical policy, separate from being a plain add; full
|
||||
precision is always correct.
|
||||
|
||||
The shard conversion methods must move any update-associated state together
|
||||
with the residual. Nonlinear updates must not claim is_plain_add.
|
||||
@@ -46,6 +53,12 @@ class ResidualUpdate(Protocol):
|
||||
# Its parameters and state remain valid after leaving the producer layer.
|
||||
# Stateful or offloaded implementations must not opt in without that guarantee.
|
||||
outlives_layer: bool
|
||||
# The producer builds the next stream itself and writes it at its exit, so
|
||||
# its exit has no sum left to complete and the previous residual is dropped.
|
||||
# Only an update applied at the exit can, and it is not a plain add.
|
||||
writes_stream: bool
|
||||
# The sum it owes may run as a quantized all-reduce.
|
||||
quantized_sum: bool
|
||||
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""Write hidden_states, the producer contribution, into residual.
|
||||
@@ -70,17 +83,23 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: Read is normalization/optional quantization without changing
|
||||
the residual, allowing compatible fused add+norm implementations.
|
||||
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
|
||||
reads_after_attn_tp_gather: Optional. The read needs every row of the
|
||||
reads_after_attn_tp_gather: The read needs every row of the
|
||||
attention's, so it runs after any gather over attention TP: a
|
||||
producer that would leave its output on this rank's attention-TP
|
||||
slice returns it to the attention's rows instead.
|
||||
completing_fusions: Optional. ReadoutFusion kernels that complete the
|
||||
sum the input owes with the residual add ahead of this read, tried
|
||||
before the boundary's own all-reduce or reduce-scatter.
|
||||
gathering_reads: Optional. Callables ``(hidden_states, residual,
|
||||
norm)`` that read the input on this rank's attention-TP slice and
|
||||
gather it over attention TP, returning ``(input, residual)``, or
|
||||
None when they do not take the batch.
|
||||
slice returns it to the attention's rows instead. False lets it
|
||||
read that slice, its result gathered after it. Not derivable from
|
||||
the rows the stage computes on: a read on the slice followed by a
|
||||
gather and a read after the gather leave the same compute rows.
|
||||
completing_fusions: ReadoutFusion kernels that complete the sum the
|
||||
input owes with the residual add ahead of this read, tried before
|
||||
the boundary's own all-reduce or reduce-scatter. Empty when the
|
||||
read offers none: the boundary's own collective followed by
|
||||
update_and_read is correct for every read.
|
||||
gathering_reads: Callables ``(hidden_states, residual, norm)`` that
|
||||
read the input on this rank's attention-TP slice and gather it over
|
||||
attention TP, returning ``(input, residual)``, or None when they do
|
||||
not take the batch. Empty when the read offers none: it then runs
|
||||
on the slice and the boundary gathers what it read.
|
||||
|
||||
init_residual initializes the stack residual. read consumes an
|
||||
already-written residual; update_and_read first applies the actual
|
||||
@@ -95,6 +114,14 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: bool
|
||||
# Preserve the read on the source rows before a DP gather.
|
||||
reads_before_dp_gather: bool
|
||||
# The read needs every row of the attention's, not this rank's slice.
|
||||
reads_after_attn_tp_gather: bool
|
||||
# Kernels that complete the sum the input owes with the residual add
|
||||
# ahead of this read; empty when the read offers none.
|
||||
completing_fusions: Tuple
|
||||
# Reads that run on this rank's attention-TP slice and gather what they
|
||||
# read; empty when the read offers none.
|
||||
gathering_reads: Tuple
|
||||
|
||||
def init_residual(self, hidden_states) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
@@ -138,6 +165,9 @@ class ResidualReadout(Protocol):
|
||||
norm: Consumer normalization module.
|
||||
quant_format: Requested input quantization format, as in read().
|
||||
post_residual_addition: Optional extra added after update, before norm.
|
||||
Each read documents where it applies, rejects or does not
|
||||
apply it. A stream that arrives already written is read with
|
||||
``read``, which is not given it.
|
||||
|
||||
Returns:
|
||||
(compute_input, residual). A fused implementation can preserve FP32
|
||||
@@ -145,6 +175,31 @@ class ResidualReadout(Protocol):
|
||||
"""
|
||||
|
||||
|
||||
class FinalRead(Protocol):
|
||||
"""The model's read of the layer stack's output, its final norm.
|
||||
|
||||
Called with the output alone on a stream its producer already wrote, else
|
||||
with the output and the residual, returning ``(normed, residual)``: the
|
||||
residual add followed by the norm, so a pending update must be a plain add.
|
||||
|
||||
Optional, each read where it is used with the default given:
|
||||
attn_tp_gather: Its own gather over attention TP of rows the stack's
|
||||
last stage leaves on this rank's attention-TP slice, returning
|
||||
None when it does not take the batch. Without it the boundary's
|
||||
gather runs, which is always correct.
|
||||
reads_attn_tp_slices: It reads this rank's attention-TP slice of the
|
||||
rows and gathers what it read itself, so the stack's last FFN
|
||||
leaves its output on that slice. False by default: the last FFN
|
||||
returns its output on the attention's rows, which any final read
|
||||
takes.
|
||||
gemma_weight: The gamma a deferred MoE finalize normalizes with, when
|
||||
the model's final norm takes that handoff.
|
||||
"""
|
||||
|
||||
def __call__(self, hidden_states, residual=None, **read_kwargs):
|
||||
"""The final norm of ``hidden_states``, after adding ``residual``."""
|
||||
|
||||
|
||||
class LayerResidualOps(NamedTuple):
|
||||
"""The reads and updates a decoder layer declares for its attention and
|
||||
its FFN."""
|
||||
|
||||
@@ -276,6 +276,8 @@ class PlainAdd:
|
||||
is_plain_add = True
|
||||
applied_at_exit = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = True
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
@@ -297,6 +299,8 @@ class ReplaceAtExit:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = True
|
||||
writes_stream = True
|
||||
quantized_sum = False
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return hidden_states
|
||||
@@ -320,10 +324,17 @@ class NormQuantReadout:
|
||||
"""The input is the residual's norm, fused with the quantization the
|
||||
consumer asks for (``quant_format``) and a post-residual addition; a plain
|
||||
add of the previous output runs in the same kernel. An empty batch skips
|
||||
the norm."""
|
||||
the norm.
|
||||
|
||||
``post_residual_addition`` is added after a plain add and before the norm.
|
||||
It is not applied by the aiter mxfp4 and fp8-group kernels, after an
|
||||
update other than a plain add, or by ``read``, which has no add."""
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather: bool = False
|
||||
reads_after_attn_tp_gather = False
|
||||
fp8_input: Optional[Fp8Input] = None
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
@@ -355,10 +366,15 @@ class NormQuantReadout:
|
||||
@dataclass(frozen=True)
|
||||
class NormReadout:
|
||||
"""The input is the residual's norm; a plain add of the previous output
|
||||
runs in the same kernel. An empty batch skips the norm."""
|
||||
runs in the same kernel. An empty batch skips the norm. It rejects a
|
||||
``quant_format`` and a ``post_residual_addition``, except that the latter
|
||||
is not applied after an update other than a plain add."""
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather: bool = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
@@ -402,6 +418,8 @@ class UnfusedNormReadout(NormReadout):
|
||||
in models that add their residual themselves. No fused kernel takes it."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
|
||||
@@ -195,6 +195,8 @@ class _BankReadout:
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
def __init__(self, state: AttnBankState):
|
||||
self.state = state
|
||||
@@ -217,7 +219,8 @@ class _BankReadout:
|
||||
|
||||
class _AttnReadout(_BankReadout):
|
||||
"""The attention input: the bank aggregation and this layer's input norm.
|
||||
A write layer snapshots the residual this read forms."""
|
||||
A write layer snapshots the residual this read forms. ``read`` rejects a
|
||||
``post_residual_addition``; ``update_and_read`` does not apply one."""
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
@@ -247,6 +250,8 @@ class _FfnReadout(_BankReadout):
|
||||
"""The FFN input: the attention output's add folded into the bank
|
||||
aggregation, and this layer's post-attention norm."""
|
||||
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
@property
|
||||
def completing_fusions(self) -> Tuple[ReadoutFusion, ...]:
|
||||
state = self.state
|
||||
|
||||
@@ -114,9 +114,12 @@ class GatedResidualState:
|
||||
|
||||
class _AttnReadout:
|
||||
"""The normalized, gated mix of the streams, from streams that already hold
|
||||
the previous layer's output: such a layer takes its input written back."""
|
||||
the previous layer's output: such a layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
@@ -150,6 +153,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
@@ -168,7 +173,10 @@ class _FfnReadout:
|
||||
"""The attention output's injection and the FFN input's mix."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
@@ -198,6 +206,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
@@ -101,10 +101,13 @@ class IHCState:
|
||||
class _AttnReadout:
|
||||
"""The gated mix of the streams and the input norm, from streams that
|
||||
already hold the previous layer's output: an iHC layer takes its input
|
||||
written back."""
|
||||
written back. A ``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -129,6 +132,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -148,7 +153,10 @@ class _FfnReadout:
|
||||
norm, fused in post_pre when the layer provides it."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -176,6 +184,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -122,10 +122,14 @@ class MHCState:
|
||||
|
||||
class _AttnReadout:
|
||||
"""hc_pre and the input norm, from streams that already hold the previous
|
||||
layer's output: an MHC layer takes its input written back."""
|
||||
layer's output: an MHC layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -149,6 +153,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -168,7 +174,10 @@ class _FfnReadout:
|
||||
in hc_ffn_post_pre when it takes the batch."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -195,6 +204,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -26,6 +26,8 @@ class PostNormAdd:
|
||||
|
||||
is_plain_add = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, norm, *, applied_at_exit: bool = False):
|
||||
self.norm = norm
|
||||
@@ -45,7 +47,10 @@ class PlainReadout:
|
||||
"""The input is the residual itself; the stage's norm is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
@@ -35,7 +35,6 @@ from sglang.srt.layers.layer_boundary.ops import sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
|
||||
@@ -281,11 +280,11 @@ class StageBoundary:
|
||||
"""Complete the sum over ``group`` that one part of this FFN's output
|
||||
owes.
|
||||
|
||||
For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from
|
||||
For an FFN that builds the next stream itself (``writes_stream``) from
|
||||
several complete parts, normalizing each before combining them; the
|
||||
stream it hands to its exit is then complete.
|
||||
"""
|
||||
if self.declaration.update is not REPLACE_AT_EXIT:
|
||||
if not self.declaration.update.writes_stream:
|
||||
raise RuntimeError(
|
||||
"only an FFN that writes the next stream itself sums its parts"
|
||||
)
|
||||
|
||||
@@ -134,7 +134,10 @@ class _ResidualMergeRead:
|
||||
dtype. The producer's update must be a plain add."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, res_scale: Optional[ResidualScaling]):
|
||||
self.res_scale = res_scale
|
||||
|
||||
@@ -837,6 +837,33 @@ class TestValuesAcrossAPipelineCut(CustomTestCase):
|
||||
self.assert_sums_completed_once(run)
|
||||
|
||||
|
||||
class TestInputScatteredAttentionInput(CustomTestCase):
|
||||
"""On an input-scattered batch an attention without a QKV hook, which
|
||||
projects the rows it is given, gets every row of its input."""
|
||||
|
||||
def test_an_attention_without_a_qkv_hook_gets_every_row(self):
|
||||
parallel = fixture.parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
|
||||
)
|
||||
with fixture.planning(parallel):
|
||||
with layer_stack():
|
||||
attention, _ = layer()
|
||||
entry = attention.plan.paths[BatchVariant.INPUT_SCATTERED].entry
|
||||
rows = torch.ones(2, 4)
|
||||
with (
|
||||
patch.object(
|
||||
boundary_prepare,
|
||||
"get_attn_tp_context",
|
||||
return_value=SimpleNamespace(is_dsa=False),
|
||||
),
|
||||
patch.object(
|
||||
boundary_prepare, "tp_gather", lambda h, fb: torch.cat([h, h])
|
||||
),
|
||||
):
|
||||
hidden = entry.attn_input_adapter(rows, SimpleNamespace(), None)
|
||||
self.assertEqual(hidden.shape[0], 4)
|
||||
|
||||
|
||||
class TestUnpaddedBatches(CustomTestCase):
|
||||
"""Without attention DP, --disable-attn-tp-gather lets a batch reach the
|
||||
stages with rows that do not divide over attention TP. Such a batch keeps
|
||||
@@ -996,6 +1023,8 @@ class TestRowsTheConsumerReads(CustomTestCase):
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
# One pipeline rank: an update other than a plain add can't cross one.
|
||||
parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1)
|
||||
@@ -1011,14 +1040,16 @@ class TestRowsTheConsumerReads(CustomTestCase):
|
||||
):
|
||||
stages = [
|
||||
stage
|
||||
for _ in range(2)
|
||||
for i in range(2)
|
||||
for stage in append_stages(
|
||||
(declare_attn(read=GatheringRead()), fixture.Norm()),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=True,
|
||||
next_layer_sparse=True,
|
||||
update=update,
|
||||
# The final read adds the last output as a
|
||||
# plain add.
|
||||
update=update if i == 0 else PLAIN_ADD,
|
||||
),
|
||||
fixture.Norm(),
|
||||
),
|
||||
|
||||
@@ -417,11 +417,14 @@ class TestPipelineResidualReception(CustomTestCase):
|
||||
from dataclasses import replace
|
||||
|
||||
from sglang.srt.layers.layer_boundary import declare_ffn
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
|
||||
stage = stub_stage(comm_instance, StageKind.ATTENTION)
|
||||
stage.declaration = replace(
|
||||
stage.declaration,
|
||||
previous=declare_ffn(update=SimpleNamespace(applied_at_exit=True)),
|
||||
previous=declare_ffn(update=REPLACE_AT_EXIT),
|
||||
)
|
||||
hidden = stage.from_pp(PPProxyTensors({"hidden_states": streams}), batch)
|
||||
residual = batch.residual_stream
|
||||
|
||||
@@ -279,7 +279,9 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
|
||||
|
||||
def test_cross_layer_update_requires_a_lifetime_guarantee(self):
|
||||
local = Layout(frozenset())
|
||||
stateful_update = SimpleNamespace(is_plain_add=False, applied_at_exit=False)
|
||||
stateful_update = SimpleNamespace(
|
||||
is_plain_add=False, applied_at_exit=False, outlives_layer=False
|
||||
)
|
||||
edge = EdgeContract(
|
||||
OutputContract(local, update=stateful_update),
|
||||
InputContract(local),
|
||||
@@ -304,6 +306,8 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
|
||||
|
||||
class ProbeRead:
|
||||
reads_before_dp_gather = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
"""A read that marks what it reads: the input is the residual plus 100.
|
||||
``is_plain_norm`` says whether it may stand in for a norm."""
|
||||
|
||||
|
||||
@@ -217,11 +217,16 @@ class TestBoundaryIntegrations(unittest.TestCase):
|
||||
|
||||
def test_an_ffn_that_completes_its_own_sum_owes_none(self):
|
||||
# Its compute completes the sum, so the exit neither sums nor defers it,
|
||||
# under DP too.
|
||||
for attn_dp in (1, 2):
|
||||
# under DP and on an input-scattered batch too.
|
||||
for attn_dp, scattered in ((1, False), (2, False), (1, True)):
|
||||
parallel = fixture.parallel_of(
|
||||
attn_dp=attn_dp,
|
||||
attn_tp=2,
|
||||
enable_attn_tp_input_scattered=scattered,
|
||||
)
|
||||
with (
|
||||
self.subTest(attn_dp=attn_dp),
|
||||
fixture.planning(fixture.parallel_of(attn_dp=attn_dp, attn_tp=2)),
|
||||
self.subTest(attn_dp=attn_dp, input_scattered=scattered),
|
||||
fixture.planning(parallel),
|
||||
):
|
||||
for sparse in (False, True):
|
||||
stages = {
|
||||
@@ -237,6 +242,10 @@ class TestBoundaryIntegrations(unittest.TestCase):
|
||||
}
|
||||
owed = stages[False].plan.paths[BatchVariant.ORDINARY].output
|
||||
self.assertIsNotNone(owed.group, sparse)
|
||||
if scattered:
|
||||
self.assertIn(
|
||||
BatchVariant.INPUT_SCATTERED, stages[True].plan.paths
|
||||
)
|
||||
for variant, path in stages[True].plan.paths.items():
|
||||
self.assertIsNone(path.output.group, (sparse, variant))
|
||||
self.assertFalse(path.output.may_defer_to_next)
|
||||
|
||||
@@ -24,10 +24,18 @@ from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
append_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import boundary as comm_boundary
|
||||
from sglang.srt.layers.layer_boundary import construction as comm_layer
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import exit as comm_exit
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import layout as comm_layout
|
||||
from sglang.srt.layers.layer_boundary import ops as transport_ops
|
||||
from sglang.srt.layers.layer_boundary import prepare as comm_ops
|
||||
@@ -48,6 +56,7 @@ from sglang.srt.layers.layer_boundary.residual import mhc as mhc_module
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_READOUT,
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
|
||||
from sglang.srt.layers.moe import utils as moe_utils
|
||||
@@ -1014,12 +1023,15 @@ class TestInputScatteredAttention(CustomTestCase):
|
||||
patch_communicator("get_parallel", lambda: parallel),
|
||||
patch_communicator(
|
||||
"get_attn_tp_context",
|
||||
lambda: SimpleNamespace(input_scattered=True),
|
||||
lambda: SimpleNamespace(input_scattered=True, is_dsa=False),
|
||||
),
|
||||
patch_communicator(
|
||||
"get_forward",
|
||||
lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False),
|
||||
),
|
||||
# This attention has no QKV hook, so it takes every row of its
|
||||
# input: the slice is gathered after the read.
|
||||
patch_communicator("tp_gather", lambda h, fb: torch.cat([h, h])),
|
||||
):
|
||||
hidden, residual = prepare_input(
|
||||
communicator.attn,
|
||||
@@ -1030,7 +1042,7 @@ class TestInputScatteredAttention(CustomTestCase):
|
||||
self.assertEqual(scattered, [4])
|
||||
# Norm: (2 * (h + r), h + r) on the slice, with h the completed sum.
|
||||
torch.testing.assert_close(residual.residual, torch.full((2, HIDDEN), 5.0))
|
||||
torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0))
|
||||
torch.testing.assert_close(hidden, torch.full((4, HIDDEN), 10.0))
|
||||
|
||||
def test_the_last_layer_completes_its_own_sum(self):
|
||||
parallel = parallel_of(
|
||||
@@ -1955,11 +1967,164 @@ class TestTwoLayers(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class RecordingGroup(Group):
|
||||
"""A Group that also records which collective each call runs."""
|
||||
|
||||
def all_reduce(self, x):
|
||||
state().ops.append("all_reduce")
|
||||
return super().all_reduce(x)
|
||||
|
||||
def reduce_scatter_tensor(self, output, input):
|
||||
state().ops.append("reduce_scatter")
|
||||
return super().reduce_scatter_tensor(output, input)
|
||||
|
||||
def all_gather_into_tensor(self, output, input):
|
||||
state().ops.append("all_gather")
|
||||
return super().all_gather_into_tensor(output, input)
|
||||
|
||||
|
||||
class TestCompleteOutputsOnInputScatteredBatches(CustomTestCase):
|
||||
"""On an input-scattered batch an FFN that hands on a complete output, one
|
||||
its compute summed or the next stream it wrote itself, gets no further
|
||||
sum, at its exit or at the next attention's entry. The next attention
|
||||
still keeps this rank's slice of the residual and takes every row of its
|
||||
input, as after an FFN whose output owes its sum."""
|
||||
|
||||
ATTN_TP = 2
|
||||
ROWS = 4
|
||||
|
||||
def run_ranks(self, output, *, reduction, qkv_hook):
|
||||
tp = self.ATTN_TP
|
||||
world = World(tp)
|
||||
WORLD[0] = world
|
||||
group = RecordingGroup(world, "tp", range(tp))
|
||||
x = torch.arange(1.0, self.ROWS * HIDDEN + 1, dtype=torch.double)
|
||||
x = x.reshape(self.ROWS, HIDDEN)
|
||||
# Norm doubles; the attention is 3x and the FFN 5x of its input.
|
||||
before_ffn = x + 3 * (2 * x)
|
||||
after_ffn = 5 * (2 * before_ffn) + before_ffn
|
||||
forward_batch = SimpleNamespace(
|
||||
forward_mode=SimpleNamespace(
|
||||
is_context_parallel_extend=lambda: False,
|
||||
is_decode_or_idle=lambda: False,
|
||||
is_extend=lambda: True,
|
||||
),
|
||||
dp_padding_mode=SimpleNamespace(is_max_len=lambda: False),
|
||||
global_dp_buffer_len=self.ROWS,
|
||||
input_ids=torch.zeros(self.ROWS),
|
||||
)
|
||||
ffn_declaration = {
|
||||
"summed by its compute": dict(output_complete=True),
|
||||
"written at its exit": dict(update=REPLACE_AT_EXIT),
|
||||
"owing its sum": {},
|
||||
}[output]
|
||||
states = [
|
||||
SimpleNamespace(
|
||||
rank=rank,
|
||||
parallel=parallel_of(
|
||||
attn_dp=1,
|
||||
attn_tp=tp,
|
||||
tp_rank=rank,
|
||||
attn_tp_rank=rank,
|
||||
tp_group=group,
|
||||
attn_tp_group=group,
|
||||
enable_attn_tp_input_scattered=True,
|
||||
),
|
||||
flags=Flags(),
|
||||
calls=[],
|
||||
ops=[],
|
||||
attn_inputs=[],
|
||||
)
|
||||
for rank in range(tp)
|
||||
]
|
||||
context = SimpleNamespace(
|
||||
input_scattered=True,
|
||||
is_dsa=False,
|
||||
set_attn_inputs=lambda inputs: state().attn_inputs.append(inputs),
|
||||
)
|
||||
|
||||
def ffn(hidden, s):
|
||||
if output == "owing its sum":
|
||||
return dense_mlp(hidden, s)
|
||||
if output == "summed by its compute":
|
||||
return 5 * hidden
|
||||
# It adds the residual itself and writes the next stream.
|
||||
return 5 * hidden + before_ffn
|
||||
|
||||
def build():
|
||||
hook = {"qkv_latent_func": lambda h, fb: 11 * h} if qkv_hook else {}
|
||||
with layer_stack():
|
||||
attention_1, ffn_1 = append_stages(
|
||||
(declare_attn(), Norm()),
|
||||
(declare_ffn(**ffn_declaration), Norm()),
|
||||
)
|
||||
attention_2, _ = append_stages(
|
||||
(declare_attn(), Norm(), hook), (declare_ffn(), Norm())
|
||||
)
|
||||
return attention_1, ffn_1, attention_2
|
||||
|
||||
def forward(rank):
|
||||
s = states[rank]
|
||||
attention_1, ffn_1, attention_2 = stages[rank]
|
||||
hidden, stream = prepare_input(
|
||||
attention_1, x * WEIGHTS[tp][rank], None, forward_batch
|
||||
)
|
||||
hidden = stream.record(
|
||||
attention(hidden, s),
|
||||
comm.PLAIN_ADD,
|
||||
declared_sum=ffn_1.entry(forward_batch).declared_sum,
|
||||
)
|
||||
hidden, stream = prepare_input(ffn_1, hidden, stream, forward_batch)
|
||||
with ffn_1.plan.output.ffn_exit(
|
||||
forward_batch, stream=ResidualStream()
|
||||
) as ffn_exit:
|
||||
hidden = ffn(hidden, s)
|
||||
computed = len(s.ops)
|
||||
hidden, stream = finish_exit(ffn_exit, hidden, stream)
|
||||
hidden, stream = prepare_input(attention_2, hidden, stream, forward_batch)
|
||||
if qkv_hook:
|
||||
hidden = s.attn_inputs[-1].fetch_qkv_latent()
|
||||
return hidden, stream.residual, s.ops[computed:]
|
||||
|
||||
with (
|
||||
running(reduce_scatterv=False, use_reduce_scatter=reduction == "rs"),
|
||||
patch_communicator("get_attn_tp_context", lambda: context),
|
||||
):
|
||||
stages = []
|
||||
for s in states:
|
||||
# Each rank binds its own stages; the stack is not shared
|
||||
# between threads.
|
||||
world.local.state = s
|
||||
stages.append(build())
|
||||
results, errors = world.run(states, forward)
|
||||
for rank, error in enumerate(errors):
|
||||
if error is not None:
|
||||
raise AssertionError(f"rank {rank} raised") from error
|
||||
rows = self.ROWS // tp
|
||||
for rank, (hidden, residual, ops) in enumerate(results):
|
||||
want = 2 * after_ffn * (11 if qkv_hook else 1)
|
||||
torch.testing.assert_close(hidden, want, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
residual, after_ffn[rank * rows : (rank + 1) * rows], rtol=0, atol=0
|
||||
)
|
||||
sums = [op for op in ops if op != "all_gather"]
|
||||
self.assertEqual(len(sums), output == "owing its sum", ops)
|
||||
|
||||
def test_a_complete_output_is_summed_no_further(self):
|
||||
for output, reduction, qkv_hook in itertools.product(
|
||||
("summed by its compute", "written at its exit", "owing its sum"),
|
||||
("ar", "rs"),
|
||||
(False, True),
|
||||
):
|
||||
with self.subTest(output=output, reduction=reduction, qkv_hook=qkv_hook):
|
||||
self.run_ranks(output, reduction=reduction, qkv_hook=qkv_hook)
|
||||
|
||||
|
||||
class TestTheFfnInputReduction(CustomTestCase):
|
||||
"""With quantized communications a prefill reduces the FFN input quantized
|
||||
for a plain residual; MHC sums its streams in full precision."""
|
||||
|
||||
def test_only_a_plain_residual_reduces_quantized(self):
|
||||
def test_only_an_update_that_allows_it_reduces_quantized(self):
|
||||
exec_ = SimpleNamespace(
|
||||
comm=SimpleNamespace(
|
||||
enable_quant_communications=True, boundary_reduction="rs+rsv"
|
||||
@@ -1968,9 +2133,11 @@ class TestTheFfnInputReduction(CustomTestCase):
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False)
|
||||
)
|
||||
for is_plain_add, reduction in ((True, "quant"), (False, "full")):
|
||||
with self.subTest(is_plain_add=is_plain_add):
|
||||
update = SimpleNamespace(is_plain_add=is_plain_add)
|
||||
for update, reduction in (
|
||||
(SimpleNamespace(quantized_sum=True), "quant"),
|
||||
(SimpleNamespace(quantized_sum=False), "full"),
|
||||
):
|
||||
with self.subTest(update=update):
|
||||
read = SimpleNamespace(
|
||||
update_and_read=lambda update, h, r, norm, **read_kwargs: (h, r)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user