[Refactor] Make residual read and update capabilities explicit and checked (#43455)

This commit is contained in:
Cheng Wan
2026-10-10 12:41:16 -07:00
committed by GitHub
parent 514feb1afc
commit b0dde87de2
19 changed files with 456 additions and 59 deletions
@@ -166,11 +166,19 @@ class _TransformedRead:
transform: OutputTransform
is_plain_norm = False
# The inner read's kernels read the stream directly and would skip the
# transform, so none is offered.
completing_fusions = ()
gathering_reads = ()
@property
def reads_before_dp_gather(self):
return self.inner.reads_before_dp_gather
@property
def reads_after_attn_tp_gather(self):
return self.inner.reads_after_attn_tp_gather
def init_residual(self, hidden_states):
return self.inner.init_residual(hidden_states)
@@ -368,8 +376,8 @@ def bind_exit(
transport. The receiver binds its read independently.
"""
update = edge.produced.update
if not getattr(update, "applied_at_exit", False):
if not getattr(update, "outlives_layer", False):
if not update.applied_at_exit:
if not update.outlives_layer:
raise NotImplementedError(
"a deferred update must guarantee its lifetime across layers"
)
@@ -396,7 +404,7 @@ def _read_fusions(read, owes, is_plain_add, *, scatters):
return ()
return tuple(
f
for f in getattr(read, "completing_fusions", ())
for f in read.completing_fusions
if f.completes is owes and f.scatters == scatters
)
@@ -558,9 +566,7 @@ def _select_entry_step(
enters_stack=enters_stack,
read=read,
read_gathers=(
getattr(read, "gathering_reads", ())
if is_plain_add or arrives_written
else ()
read.gathering_reads if is_plain_add or arrives_written else ()
),
),
input_move=(
@@ -313,6 +313,14 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
raise ValueError("connections do not match the stage declaration")
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
update = declaration.update
if declaration.terminal and not (update.is_plain_add or update.applied_at_exit):
# The final read adds the last output into the residual itself, as a
# plain add, before its norm.
raise NotImplementedError(
f"the layer stack ends on a stage whose {type(update).__name__} "
"update the final read would apply as a plain add"
)
variants = {}
for variant, edge in incoming.entries.items():
if (
@@ -321,13 +329,17 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
in edge.produced.layout.sharded - edge.need.layout.sharded
and not _cp_gathers_over_attn_cp()
):
raise NotImplementedError("MHC with a gather over attention CP")
raise NotImplementedError(
"an update applied at the stage's exit with a gather over attention CP"
)
attn_input_adapter = None
if declaration.kind is StageKind.ATTENTION:
# On an input-scattered batch the step gathers the rows itself for
# an attention whose QKV hook does not gather them after the
# projection.
attn_input_adapter = (
_attn_input_scattered
if variant is BatchVariant.INPUT_SCATTERED
and not declaration.update.is_plain_add
else _attn_input_default
)
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
@@ -271,7 +271,7 @@ class ExitPolicy:
hidden_states,
steps.output.group,
forward_batch,
may_quantize=steps.output.update.is_plain_add,
may_quantize=steps.output.update.quantized_sum,
)
if steps.output.transform is not None:
hidden_states = steps.output.transform.apply(hidden_states)
@@ -476,7 +476,7 @@ class MixerExit:
hidden_states,
self._group,
self._forward_batch,
may_quantize=self._update.is_plain_add,
may_quantize=self._update.quantized_sum,
)
return self._stream.record(
hidden_states, self._update, declared_sum=self._declared_sum
@@ -45,12 +45,14 @@ from sglang.srt.layers.layer_boundary.layout import (
token_axis_sizes,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual import (
ResidualReadout,
ResidualUpdate,
)
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_QUANT_READOUT,
NORM_READOUT,
PLAIN_ADD,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
@@ -150,6 +152,8 @@ def _resolve_ffn(
group = SumGroup.MOE_OUTPUT
else:
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
# An FFN that writes the next stream itself hands on a complete output.
complete = on_rank_rows or update.writes_stream or output_complete
if variant is BatchVariant.SEQUENCE_PARALLEL:
return (
StageContract(
@@ -172,8 +176,9 @@ def _resolve_ffn(
InputContract(full, read=read),
OutputContract(
attention,
group=group,
may_reduce_scatter=use_reduce_scatter
group=None if complete else group,
may_reduce_scatter=not complete
and use_reduce_scatter
and (scattered_residual or not terminal),
update=update,
transform=output_transform,
@@ -199,8 +204,6 @@ def _resolve_ffn(
rows = Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
)
# An FFN that writes the next stream itself hands on a complete output.
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
produced = (
OutputContract(rows, update=update, transform=output_transform)
if complete
@@ -301,6 +304,51 @@ class StageDeclaration:
for source in (self.previous, self.prepared_from):
if source is not None and not isinstance(source, StageDeclaration):
raise TypeError("stage sources must be declarations")
_check_update(self.update)
_check_read(self.read)
# The members every ResidualUpdate and ResidualReadout must set: the binding
# reads each of them, and none has a default that is correct for every
# implementation.
_UPDATE_FACTS = (
"is_plain_add",
"applied_at_exit",
"outlives_layer",
"writes_stream",
"quantized_sum",
)
_READ_FACTS = ("is_plain_norm", "reads_before_dp_gather", "reads_after_attn_tp_gather")
# The kernels a read supplies, which the binding also reads; empty when it
# supplies none.
_READ_KERNELS = ("completing_fusions", "gathering_reads")
def _require(protocol, implementation, facts):
missing = [name for name in facts if not hasattr(implementation, name)]
if missing:
raise TypeError(
f"{type(implementation).__name__} does not set {', '.join(missing)}, "
f"which every {protocol} must"
)
def _check_update(update):
_require("ResidualUpdate", update, _UPDATE_FACTS)
name = type(update).__name__
if update.writes_stream and (update.is_plain_add or not update.applied_at_exit):
raise ValueError(
f"{name} writes the next stream itself, which only an update "
"applied at its exit and other than a plain add does"
)
if update.quantized_sum and not update.is_plain_add:
raise ValueError(
f"{name} lets its sum run quantized, which only a plain add does"
)
def _check_read(read):
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS)
@dataclass(frozen=True)
@@ -420,11 +468,13 @@ def _resolve_stage(stage, variant, following=None):
if stage.update.applied_at_exit:
if stage.sparse and moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
"an update applied at the stage's exit with a MoE gathered over "
"the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_scattered_possible():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
"an update applied at the stage's exit with input-scattered "
"attention under attention CP"
)
if stage.kind is StageKind.FFN:
declaration, residual, returned = _resolve_ffn(
@@ -439,8 +489,7 @@ def _resolve_stage(stage, variant, following=None):
)
exit_rows = resolve_exit_rows(stage.exit_rows)
if exit_rows is ExitRows.ATTENTION or (
following is not None
and getattr(following.read, "reads_after_attn_tp_gather", False)
following is not None and following.read.reads_after_attn_tp_gather
):
returned = attention
elif exit_rows is ExitRows.SLICE:
@@ -706,11 +755,12 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
stack's first. What they build is discarded.
next_layers: Likewise for the layers after this stack, whose first
declared stage is the consumer of this stack's last.
final_read: The model's final read of the stack's output, when it is
not a plain final norm: the consumer of the last stage when no
later layer declares one. Its ``attn_tp_gather`` gathers the rows
that stage leaves on this rank's attention-TP slice, and with
``reads_attn_tp_slices`` it reads that slice instead.
final_read: The model's final read of the stack's output (a
``FinalRead``), when it is not a plain final norm: the consumer of
the last stage when no later layer declares one. Its
``attn_tp_gather`` gathers the rows that stage leaves on this
rank's attention-TP slice, and with ``reads_attn_tp_slices`` it
reads that slice instead.
"""
global _stack
outer = _stack
@@ -135,9 +135,8 @@ def _reduce_update_read(
if result is not None:
return result
if group is SumGroup.ATTN_TP:
# MHC sums its streams in full precision.
hidden_states = attn_tp_all_reduce(
hidden_states, forward_batch, may_quantize=update.is_plain_add
hidden_states, forward_batch, may_quantize=update.quantized_sum
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
@@ -31,6 +31,13 @@ class ResidualUpdate(Protocol):
than leaving the operation for the next prepare.
outlives_layer: Whether parameters/state remain valid after
leaving the producer layer, including any offload or state reuse.
writes_stream: Whether the producer computes the next stream itself
and writes it at its exit, dropping the previous residual: what
reaches its exit is already complete.
quantized_sum: Whether the sum the contribution owes may be completed
by a quantized all-reduce when quantized communication is enabled.
This is a numerical policy, separate from being a plain add; full
precision is always correct.
The shard conversion methods must move any update-associated state together
with the residual. Nonlinear updates must not claim is_plain_add.
@@ -46,6 +53,12 @@ class ResidualUpdate(Protocol):
# Its parameters and state remain valid after leaving the producer layer.
# Stateful or offloaded implementations must not opt in without that guarantee.
outlives_layer: bool
# The producer builds the next stream itself and writes it at its exit, so
# its exit has no sum left to complete and the previous residual is dropped.
# Only an update applied at the exit can, and it is not a plain add.
writes_stream: bool
# The sum it owes may run as a quantized all-reduce.
quantized_sum: bool
def update(self, hidden_states, residual) -> torch.Tensor:
"""Write hidden_states, the producer contribution, into residual.
@@ -70,17 +83,23 @@ class ResidualReadout(Protocol):
is_plain_norm: Read is normalization/optional quantization without changing
the residual, allowing compatible fused add+norm implementations.
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
reads_after_attn_tp_gather: Optional. The read needs every row of the
reads_after_attn_tp_gather: The read needs every row of the
attention's, so it runs after any gather over attention TP: a
producer that would leave its output on this rank's attention-TP
slice returns it to the attention's rows instead.
completing_fusions: Optional. ReadoutFusion kernels that complete the
sum the input owes with the residual add ahead of this read, tried
before the boundary's own all-reduce or reduce-scatter.
gathering_reads: Optional. Callables ``(hidden_states, residual,
norm)`` that read the input on this rank's attention-TP slice and
gather it over attention TP, returning ``(input, residual)``, or
None when they do not take the batch.
slice returns it to the attention's rows instead. False lets it
read that slice, its result gathered after it. Not derivable from
the rows the stage computes on: a read on the slice followed by a
gather and a read after the gather leave the same compute rows.
completing_fusions: ReadoutFusion kernels that complete the sum the
input owes with the residual add ahead of this read, tried before
the boundary's own all-reduce or reduce-scatter. Empty when the
read offers none: the boundary's own collective followed by
update_and_read is correct for every read.
gathering_reads: Callables ``(hidden_states, residual, norm)`` that
read the input on this rank's attention-TP slice and gather it over
attention TP, returning ``(input, residual)``, or None when they do
not take the batch. Empty when the read offers none: it then runs
on the slice and the boundary gathers what it read.
init_residual initializes the stack residual. read consumes an
already-written residual; update_and_read first applies the actual
@@ -95,6 +114,14 @@ class ResidualReadout(Protocol):
is_plain_norm: bool
# Preserve the read on the source rows before a DP gather.
reads_before_dp_gather: bool
# The read needs every row of the attention's, not this rank's slice.
reads_after_attn_tp_gather: bool
# Kernels that complete the sum the input owes with the residual add
# ahead of this read; empty when the read offers none.
completing_fusions: Tuple
# Reads that run on this rank's attention-TP slice and gather what they
# read; empty when the read offers none.
gathering_reads: Tuple
def init_residual(self, hidden_states) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
@@ -138,6 +165,9 @@ class ResidualReadout(Protocol):
norm: Consumer normalization module.
quant_format: Requested input quantization format, as in read().
post_residual_addition: Optional extra added after update, before norm.
Each read documents where it applies, rejects or does not
apply it. A stream that arrives already written is read with
``read``, which is not given it.
Returns:
(compute_input, residual). A fused implementation can preserve FP32
@@ -145,6 +175,31 @@ class ResidualReadout(Protocol):
"""
class FinalRead(Protocol):
"""The model's read of the layer stack's output, its final norm.
Called with the output alone on a stream its producer already wrote, else
with the output and the residual, returning ``(normed, residual)``: the
residual add followed by the norm, so a pending update must be a plain add.
Optional, each read where it is used with the default given:
attn_tp_gather: Its own gather over attention TP of rows the stack's
last stage leaves on this rank's attention-TP slice, returning
None when it does not take the batch. Without it the boundary's
gather runs, which is always correct.
reads_attn_tp_slices: It reads this rank's attention-TP slice of the
rows and gathers what it read itself, so the stack's last FFN
leaves its output on that slice. False by default: the last FFN
returns its output on the attention's rows, which any final read
takes.
gemma_weight: The gamma a deferred MoE finalize normalizes with, when
the model's final norm takes that handoff.
"""
def __call__(self, hidden_states, residual=None, **read_kwargs):
"""The final norm of ``hidden_states``, after adding ``residual``."""
class LayerResidualOps(NamedTuple):
"""The reads and updates a decoder layer declares for its attention and
its FFN."""
@@ -276,6 +276,8 @@ class PlainAdd:
is_plain_add = True
applied_at_exit = False
outlives_layer = True
writes_stream = False
quantized_sum = True
def update(self, hidden_states, residual):
hidden_states += residual
@@ -297,6 +299,8 @@ class ReplaceAtExit:
is_plain_add = False
applied_at_exit = True
outlives_layer = True
writes_stream = True
quantized_sum = False
def update(self, hidden_states, residual):
return hidden_states
@@ -320,10 +324,17 @@ class NormQuantReadout:
"""The input is the residual's norm, fused with the quantization the
consumer asks for (``quant_format``) and a post-residual addition; a plain
add of the previous output runs in the same kernel. An empty batch skips
the norm."""
the norm.
``post_residual_addition`` is added after a plain add and before the norm.
It is not applied by the aiter mxfp4 and fp8-group kernels, after an
update other than a plain add, or by ``read``, which has no add."""
is_plain_norm = True
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather: bool = False
reads_after_attn_tp_gather = False
fp8_input: Optional[Fp8Input] = None
def init_residual(self, hidden_states):
@@ -355,10 +366,15 @@ class NormQuantReadout:
@dataclass(frozen=True)
class NormReadout:
"""The input is the residual's norm; a plain add of the previous output
runs in the same kernel. An empty batch skips the norm."""
runs in the same kernel. An empty batch skips the norm. It rejects a
``quant_format`` and a ``post_residual_addition``, except that the latter
is not applied after an update other than a plain add."""
is_plain_norm = True
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather: bool = False
reads_after_attn_tp_gather = False
def init_residual(self, hidden_states):
return hidden_states
@@ -402,6 +418,8 @@ class UnfusedNormReadout(NormReadout):
in models that add their residual themselves. No fused kernel takes it."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
def update_and_read(
self,
@@ -195,6 +195,8 @@ class _BankReadout:
is_plain_norm = False
reads_before_dp_gather = True
completing_fusions = ()
gathering_reads = ()
def __init__(self, state: AttnBankState):
self.state = state
@@ -217,7 +219,8 @@ class _BankReadout:
class _AttnReadout(_BankReadout):
"""The attention input: the bank aggregation and this layer's input norm.
A write layer snapshots the residual this read forms."""
A write layer snapshots the residual this read forms. ``read`` rejects a
``post_residual_addition``; ``update_and_read`` does not apply one."""
@property
def reads_after_attn_tp_gather(self):
@@ -247,6 +250,8 @@ class _FfnReadout(_BankReadout):
"""The FFN input: the attention output's add folded into the bank
aggregation, and this layer's post-attention norm."""
reads_after_attn_tp_gather = False
@property
def completing_fusions(self) -> Tuple[ReadoutFusion, ...]:
state = self.state
@@ -114,9 +114,12 @@ class GatedResidualState:
class _AttnReadout:
"""The normalized, gated mix of the streams, from streams that already hold
the previous layer's output: such a layer takes its input written back."""
the previous layer's output: such a layer takes its input written back. A
``post_residual_addition`` is not applied."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
def __init__(self, state: GatedResidualState):
@@ -150,6 +153,8 @@ class _AttnUpdate:
is_plain_add = False
applied_at_exit = False
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: GatedResidualState):
self.state = state
@@ -168,7 +173,10 @@ class _FfnReadout:
"""The attention output's injection and the FFN input's mix."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, state: GatedResidualState):
self.state = state
@@ -198,6 +206,8 @@ class _FfnUpdate:
is_plain_add = False
applied_at_exit = True
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: GatedResidualState):
self.state = state
@@ -101,10 +101,13 @@ class IHCState:
class _AttnReadout:
"""The gated mix of the streams and the input norm, from streams that
already hold the previous layer's output: an iHC layer takes its input
written back."""
written back. A ``post_residual_addition`` is not applied."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, state: IHCState):
self.state = state
@@ -129,6 +132,8 @@ class _AttnUpdate:
is_plain_add = False
applied_at_exit = False
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: IHCState):
self.state = state
@@ -148,7 +153,10 @@ class _FfnReadout:
norm, fused in post_pre when the layer provides it."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, state: IHCState):
self.state = state
@@ -176,6 +184,8 @@ class _FfnUpdate:
is_plain_add = False
applied_at_exit = True
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: IHCState):
self.state = state
@@ -122,10 +122,14 @@ class MHCState:
class _AttnReadout:
"""hc_pre and the input norm, from streams that already hold the previous
layer's output: an MHC layer takes its input written back."""
layer's output: an MHC layer takes its input written back. A
``post_residual_addition`` is not applied."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, state: MHCState):
self.state = state
@@ -149,6 +153,8 @@ class _AttnUpdate:
is_plain_add = False
applied_at_exit = False
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: MHCState):
self.state = state
@@ -168,7 +174,10 @@ class _FfnReadout:
in hc_ffn_post_pre when it takes the batch."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, state: MHCState):
self.state = state
@@ -195,6 +204,8 @@ class _FfnUpdate:
is_plain_add = False
applied_at_exit = True
outlives_layer = False
writes_stream = False
quantized_sum = False
def __init__(self, state: MHCState):
self.state = state
@@ -26,6 +26,8 @@ class PostNormAdd:
is_plain_add = False
outlives_layer = True
writes_stream = False
quantized_sum = False
def __init__(self, norm, *, applied_at_exit: bool = False):
self.norm = norm
@@ -45,7 +47,10 @@ class PlainReadout:
"""The input is the residual itself; the stage's norm is not applied."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def init_residual(self, hidden_states):
return hidden_states
@@ -35,7 +35,6 @@ from sglang.srt.layers.layer_boundary.ops import sum_output
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
@@ -281,11 +280,11 @@ class StageBoundary:
"""Complete the sum over ``group`` that one part of this FFN's output
owes.
For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from
For an FFN that builds the next stream itself (``writes_stream``) from
several complete parts, normalizing each before combining them; the
stream it hands to its exit is then complete.
"""
if self.declaration.update is not REPLACE_AT_EXIT:
if not self.declaration.update.writes_stream:
raise RuntimeError(
"only an FFN that writes the next stream itself sums its parts"
)
+3
View File
@@ -134,7 +134,10 @@ class _ResidualMergeRead:
dtype. The producer's update must be a plain add."""
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
reads_before_dp_gather = False
reads_after_attn_tp_gather = False
def __init__(self, res_scale: Optional[ResidualScaling]):
self.res_scale = res_scale
@@ -837,6 +837,33 @@ class TestValuesAcrossAPipelineCut(CustomTestCase):
self.assert_sums_completed_once(run)
class TestInputScatteredAttentionInput(CustomTestCase):
"""On an input-scattered batch an attention without a QKV hook, which
projects the rows it is given, gets every row of its input."""
def test_an_attention_without_a_qkv_hook_gets_every_row(self):
parallel = fixture.parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
with fixture.planning(parallel):
with layer_stack():
attention, _ = layer()
entry = attention.plan.paths[BatchVariant.INPUT_SCATTERED].entry
rows = torch.ones(2, 4)
with (
patch.object(
boundary_prepare,
"get_attn_tp_context",
return_value=SimpleNamespace(is_dsa=False),
),
patch.object(
boundary_prepare, "tp_gather", lambda h, fb: torch.cat([h, h])
),
):
hidden = entry.attn_input_adapter(rows, SimpleNamespace(), None)
self.assertEqual(hidden.shape[0], 4)
class TestUnpaddedBatches(CustomTestCase):
"""Without attention DP, --disable-attn-tp-gather lets a batch reach the
stages with rows that do not divide over attention TP. Such a batch keeps
@@ -996,6 +1023,8 @@ class TestRowsTheConsumerReads(CustomTestCase):
is_plain_add = False
applied_at_exit = False
outlives_layer = True
writes_stream = False
quantized_sum = False
# One pipeline rank: an update other than a plain add can't cross one.
parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1)
@@ -1011,14 +1040,16 @@ class TestRowsTheConsumerReads(CustomTestCase):
):
stages = [
stage
for _ in range(2)
for i in range(2)
for stage in append_stages(
(declare_attn(read=GatheringRead()), fixture.Norm()),
(
declare_ffn(
sparse=True,
next_layer_sparse=True,
update=update,
# The final read adds the last output as a
# plain add.
update=update if i == 0 else PLAIN_ADD,
),
fixture.Norm(),
),
@@ -417,11 +417,14 @@ class TestPipelineResidualReception(CustomTestCase):
from dataclasses import replace
from sglang.srt.layers.layer_boundary import declare_ffn
from sglang.srt.layers.layer_boundary.residual.add_norm import (
REPLACE_AT_EXIT,
)
stage = stub_stage(comm_instance, StageKind.ATTENTION)
stage.declaration = replace(
stage.declaration,
previous=declare_ffn(update=SimpleNamespace(applied_at_exit=True)),
previous=declare_ffn(update=REPLACE_AT_EXIT),
)
hidden = stage.from_pp(PPProxyTensors({"hidden_states": streams}), batch)
residual = batch.residual_stream
@@ -279,7 +279,9 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
def test_cross_layer_update_requires_a_lifetime_guarantee(self):
local = Layout(frozenset())
stateful_update = SimpleNamespace(is_plain_add=False, applied_at_exit=False)
stateful_update = SimpleNamespace(
is_plain_add=False, applied_at_exit=False, outlives_layer=False
)
edge = EdgeContract(
OutputContract(local, update=stateful_update),
InputContract(local),
@@ -304,6 +306,8 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase):
class ProbeRead:
reads_before_dp_gather = False
completing_fusions = ()
gathering_reads = ()
"""A read that marks what it reads: the input is the residual plus 100.
``is_plain_norm`` says whether it may stand in for a norm."""
@@ -217,11 +217,16 @@ class TestBoundaryIntegrations(unittest.TestCase):
def test_an_ffn_that_completes_its_own_sum_owes_none(self):
# Its compute completes the sum, so the exit neither sums nor defers it,
# under DP too.
for attn_dp in (1, 2):
# under DP and on an input-scattered batch too.
for attn_dp, scattered in ((1, False), (2, False), (1, True)):
parallel = fixture.parallel_of(
attn_dp=attn_dp,
attn_tp=2,
enable_attn_tp_input_scattered=scattered,
)
with (
self.subTest(attn_dp=attn_dp),
fixture.planning(fixture.parallel_of(attn_dp=attn_dp, attn_tp=2)),
self.subTest(attn_dp=attn_dp, input_scattered=scattered),
fixture.planning(parallel),
):
for sparse in (False, True):
stages = {
@@ -237,6 +242,10 @@ class TestBoundaryIntegrations(unittest.TestCase):
}
owed = stages[False].plan.paths[BatchVariant.ORDINARY].output
self.assertIsNotNone(owed.group, sparse)
if scattered:
self.assertIn(
BatchVariant.INPUT_SCATTERED, stages[True].plan.paths
)
for variant, path in stages[True].plan.paths.items():
self.assertIsNone(path.output.group, (sparse, variant))
self.assertFalse(path.output.may_defer_to_next)
@@ -24,10 +24,18 @@ from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.layer_boundary import (
SumGroup,
TokenAxis,
append_stages,
)
from sglang.srt.layers.layer_boundary import boundary as comm_boundary
from sglang.srt.layers.layer_boundary import construction as comm_layer
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
)
from sglang.srt.layers.layer_boundary import exit as comm_exit
from sglang.srt.layers.layer_boundary import (
layer_stack,
)
from sglang.srt.layers.layer_boundary import layout as comm_layout
from sglang.srt.layers.layer_boundary import ops as transport_ops
from sglang.srt.layers.layer_boundary import prepare as comm_ops
@@ -48,6 +56,7 @@ from sglang.srt.layers.layer_boundary.residual import mhc as mhc_module
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_READOUT,
PLAIN_RESIDUAL_OPS,
REPLACE_AT_EXIT,
)
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
from sglang.srt.layers.moe import utils as moe_utils
@@ -1014,12 +1023,15 @@ class TestInputScatteredAttention(CustomTestCase):
patch_communicator("get_parallel", lambda: parallel),
patch_communicator(
"get_attn_tp_context",
lambda: SimpleNamespace(input_scattered=True),
lambda: SimpleNamespace(input_scattered=True, is_dsa=False),
),
patch_communicator(
"get_forward",
lambda: SimpleNamespace(sp_active=False, attn_input_scattered=False),
),
# This attention has no QKV hook, so it takes every row of its
# input: the slice is gathered after the read.
patch_communicator("tp_gather", lambda h, fb: torch.cat([h, h])),
):
hidden, residual = prepare_input(
communicator.attn,
@@ -1030,7 +1042,7 @@ class TestInputScatteredAttention(CustomTestCase):
self.assertEqual(scattered, [4])
# Norm: (2 * (h + r), h + r) on the slice, with h the completed sum.
torch.testing.assert_close(residual.residual, torch.full((2, HIDDEN), 5.0))
torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0))
torch.testing.assert_close(hidden, torch.full((4, HIDDEN), 10.0))
def test_the_last_layer_completes_its_own_sum(self):
parallel = parallel_of(
@@ -1955,11 +1967,164 @@ class TestTwoLayers(CustomTestCase):
)
class RecordingGroup(Group):
"""A Group that also records which collective each call runs."""
def all_reduce(self, x):
state().ops.append("all_reduce")
return super().all_reduce(x)
def reduce_scatter_tensor(self, output, input):
state().ops.append("reduce_scatter")
return super().reduce_scatter_tensor(output, input)
def all_gather_into_tensor(self, output, input):
state().ops.append("all_gather")
return super().all_gather_into_tensor(output, input)
class TestCompleteOutputsOnInputScatteredBatches(CustomTestCase):
"""On an input-scattered batch an FFN that hands on a complete output, one
its compute summed or the next stream it wrote itself, gets no further
sum, at its exit or at the next attention's entry. The next attention
still keeps this rank's slice of the residual and takes every row of its
input, as after an FFN whose output owes its sum."""
ATTN_TP = 2
ROWS = 4
def run_ranks(self, output, *, reduction, qkv_hook):
tp = self.ATTN_TP
world = World(tp)
WORLD[0] = world
group = RecordingGroup(world, "tp", range(tp))
x = torch.arange(1.0, self.ROWS * HIDDEN + 1, dtype=torch.double)
x = x.reshape(self.ROWS, HIDDEN)
# Norm doubles; the attention is 3x and the FFN 5x of its input.
before_ffn = x + 3 * (2 * x)
after_ffn = 5 * (2 * before_ffn) + before_ffn
forward_batch = SimpleNamespace(
forward_mode=SimpleNamespace(
is_context_parallel_extend=lambda: False,
is_decode_or_idle=lambda: False,
is_extend=lambda: True,
),
dp_padding_mode=SimpleNamespace(is_max_len=lambda: False),
global_dp_buffer_len=self.ROWS,
input_ids=torch.zeros(self.ROWS),
)
ffn_declaration = {
"summed by its compute": dict(output_complete=True),
"written at its exit": dict(update=REPLACE_AT_EXIT),
"owing its sum": {},
}[output]
states = [
SimpleNamespace(
rank=rank,
parallel=parallel_of(
attn_dp=1,
attn_tp=tp,
tp_rank=rank,
attn_tp_rank=rank,
tp_group=group,
attn_tp_group=group,
enable_attn_tp_input_scattered=True,
),
flags=Flags(),
calls=[],
ops=[],
attn_inputs=[],
)
for rank in range(tp)
]
context = SimpleNamespace(
input_scattered=True,
is_dsa=False,
set_attn_inputs=lambda inputs: state().attn_inputs.append(inputs),
)
def ffn(hidden, s):
if output == "owing its sum":
return dense_mlp(hidden, s)
if output == "summed by its compute":
return 5 * hidden
# It adds the residual itself and writes the next stream.
return 5 * hidden + before_ffn
def build():
hook = {"qkv_latent_func": lambda h, fb: 11 * h} if qkv_hook else {}
with layer_stack():
attention_1, ffn_1 = append_stages(
(declare_attn(), Norm()),
(declare_ffn(**ffn_declaration), Norm()),
)
attention_2, _ = append_stages(
(declare_attn(), Norm(), hook), (declare_ffn(), Norm())
)
return attention_1, ffn_1, attention_2
def forward(rank):
s = states[rank]
attention_1, ffn_1, attention_2 = stages[rank]
hidden, stream = prepare_input(
attention_1, x * WEIGHTS[tp][rank], None, forward_batch
)
hidden = stream.record(
attention(hidden, s),
comm.PLAIN_ADD,
declared_sum=ffn_1.entry(forward_batch).declared_sum,
)
hidden, stream = prepare_input(ffn_1, hidden, stream, forward_batch)
with ffn_1.plan.output.ffn_exit(
forward_batch, stream=ResidualStream()
) as ffn_exit:
hidden = ffn(hidden, s)
computed = len(s.ops)
hidden, stream = finish_exit(ffn_exit, hidden, stream)
hidden, stream = prepare_input(attention_2, hidden, stream, forward_batch)
if qkv_hook:
hidden = s.attn_inputs[-1].fetch_qkv_latent()
return hidden, stream.residual, s.ops[computed:]
with (
running(reduce_scatterv=False, use_reduce_scatter=reduction == "rs"),
patch_communicator("get_attn_tp_context", lambda: context),
):
stages = []
for s in states:
# Each rank binds its own stages; the stack is not shared
# between threads.
world.local.state = s
stages.append(build())
results, errors = world.run(states, forward)
for rank, error in enumerate(errors):
if error is not None:
raise AssertionError(f"rank {rank} raised") from error
rows = self.ROWS // tp
for rank, (hidden, residual, ops) in enumerate(results):
want = 2 * after_ffn * (11 if qkv_hook else 1)
torch.testing.assert_close(hidden, want, rtol=0, atol=0)
torch.testing.assert_close(
residual, after_ffn[rank * rows : (rank + 1) * rows], rtol=0, atol=0
)
sums = [op for op in ops if op != "all_gather"]
self.assertEqual(len(sums), output == "owing its sum", ops)
def test_a_complete_output_is_summed_no_further(self):
for output, reduction, qkv_hook in itertools.product(
("summed by its compute", "written at its exit", "owing its sum"),
("ar", "rs"),
(False, True),
):
with self.subTest(output=output, reduction=reduction, qkv_hook=qkv_hook):
self.run_ranks(output, reduction=reduction, qkv_hook=qkv_hook)
class TestTheFfnInputReduction(CustomTestCase):
"""With quantized communications a prefill reduces the FFN input quantized
for a plain residual; MHC sums its streams in full precision."""
def test_only_a_plain_residual_reduces_quantized(self):
def test_only_an_update_that_allows_it_reduces_quantized(self):
exec_ = SimpleNamespace(
comm=SimpleNamespace(
enable_quant_communications=True, boundary_reduction="rs+rsv"
@@ -1968,9 +2133,11 @@ class TestTheFfnInputReduction(CustomTestCase):
batch = SimpleNamespace(
forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False)
)
for is_plain_add, reduction in ((True, "quant"), (False, "full")):
with self.subTest(is_plain_add=is_plain_add):
update = SimpleNamespace(is_plain_add=is_plain_add)
for update, reduction in (
(SimpleNamespace(quantized_sum=True), "quant"),
(SimpleNamespace(quantized_sum=False), "full"),
):
with self.subTest(update=update):
read = SimpleNamespace(
update_and_read=lambda update, h, r, norm, **read_kwargs: (h, r)
)