[Refactor] Make residual read and update capabilities explicit and checked (#43455)
This commit is contained in:
@@ -166,11 +166,19 @@ class _TransformedRead:
|
||||
transform: OutputTransform
|
||||
|
||||
is_plain_norm = False
|
||||
# The inner read's kernels read the stream directly and would skip the
|
||||
# transform, so none is offered.
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
@property
|
||||
def reads_before_dp_gather(self):
|
||||
return self.inner.reads_before_dp_gather
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
return self.inner.reads_after_attn_tp_gather
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.inner.init_residual(hidden_states)
|
||||
|
||||
@@ -368,8 +376,8 @@ def bind_exit(
|
||||
transport. The receiver binds its read independently.
|
||||
"""
|
||||
update = edge.produced.update
|
||||
if not getattr(update, "applied_at_exit", False):
|
||||
if not getattr(update, "outlives_layer", False):
|
||||
if not update.applied_at_exit:
|
||||
if not update.outlives_layer:
|
||||
raise NotImplementedError(
|
||||
"a deferred update must guarantee its lifetime across layers"
|
||||
)
|
||||
@@ -396,7 +404,7 @@ def _read_fusions(read, owes, is_plain_add, *, scatters):
|
||||
return ()
|
||||
return tuple(
|
||||
f
|
||||
for f in getattr(read, "completing_fusions", ())
|
||||
for f in read.completing_fusions
|
||||
if f.completes is owes and f.scatters == scatters
|
||||
)
|
||||
|
||||
@@ -558,9 +566,7 @@ def _select_entry_step(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
read_gathers=(
|
||||
getattr(read, "gathering_reads", ())
|
||||
if is_plain_add or arrives_written
|
||||
else ()
|
||||
read.gathering_reads if is_plain_add or arrives_written else ()
|
||||
),
|
||||
),
|
||||
input_move=(
|
||||
|
||||
@@ -313,6 +313,14 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
update = declaration.update
|
||||
if declaration.terminal and not (update.is_plain_add or update.applied_at_exit):
|
||||
# The final read adds the last output into the residual itself, as a
|
||||
# plain add, before its norm.
|
||||
raise NotImplementedError(
|
||||
f"the layer stack ends on a stage whose {type(update).__name__} "
|
||||
"update the final read would apply as a plain add"
|
||||
)
|
||||
variants = {}
|
||||
for variant, edge in incoming.entries.items():
|
||||
if (
|
||||
@@ -321,13 +329,17 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with a gather over attention CP"
|
||||
)
|
||||
attn_input_adapter = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
# On an input-scattered batch the step gathers the rows itself for
|
||||
# an attention whose QKV hook does not gather them after the
|
||||
# projection.
|
||||
attn_input_adapter = (
|
||||
_attn_input_scattered
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and not declaration.update.is_plain_add
|
||||
else _attn_input_default
|
||||
)
|
||||
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
|
||||
|
||||
@@ -271,7 +271,7 @@ class ExitPolicy:
|
||||
hidden_states,
|
||||
steps.output.group,
|
||||
forward_batch,
|
||||
may_quantize=steps.output.update.is_plain_add,
|
||||
may_quantize=steps.output.update.quantized_sum,
|
||||
)
|
||||
if steps.output.transform is not None:
|
||||
hidden_states = steps.output.transform.apply(hidden_states)
|
||||
@@ -476,7 +476,7 @@ class MixerExit:
|
||||
hidden_states,
|
||||
self._group,
|
||||
self._forward_batch,
|
||||
may_quantize=self._update.is_plain_add,
|
||||
may_quantize=self._update.quantized_sum,
|
||||
)
|
||||
return self._stream.record(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
|
||||
@@ -45,12 +45,14 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
ResidualReadout,
|
||||
ResidualUpdate,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
NORM_QUANT_READOUT,
|
||||
NORM_READOUT,
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
@@ -150,6 +152,8 @@ def _resolve_ffn(
|
||||
group = SumGroup.MOE_OUTPUT
|
||||
else:
|
||||
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update.writes_stream or output_complete
|
||||
if variant is BatchVariant.SEQUENCE_PARALLEL:
|
||||
return (
|
||||
StageContract(
|
||||
@@ -172,8 +176,9 @@ def _resolve_ffn(
|
||||
InputContract(full, read=read),
|
||||
OutputContract(
|
||||
attention,
|
||||
group=group,
|
||||
may_reduce_scatter=use_reduce_scatter
|
||||
group=None if complete else group,
|
||||
may_reduce_scatter=not complete
|
||||
and use_reduce_scatter
|
||||
and (scattered_residual or not terminal),
|
||||
update=update,
|
||||
transform=output_transform,
|
||||
@@ -199,8 +204,6 @@ def _resolve_ffn(
|
||||
rows = Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
|
||||
produced = (
|
||||
OutputContract(rows, update=update, transform=output_transform)
|
||||
if complete
|
||||
@@ -301,6 +304,51 @@ class StageDeclaration:
|
||||
for source in (self.previous, self.prepared_from):
|
||||
if source is not None and not isinstance(source, StageDeclaration):
|
||||
raise TypeError("stage sources must be declarations")
|
||||
_check_update(self.update)
|
||||
_check_read(self.read)
|
||||
|
||||
|
||||
# The members every ResidualUpdate and ResidualReadout must set: the binding
|
||||
# reads each of them, and none has a default that is correct for every
|
||||
# implementation.
|
||||
_UPDATE_FACTS = (
|
||||
"is_plain_add",
|
||||
"applied_at_exit",
|
||||
"outlives_layer",
|
||||
"writes_stream",
|
||||
"quantized_sum",
|
||||
)
|
||||
_READ_FACTS = ("is_plain_norm", "reads_before_dp_gather", "reads_after_attn_tp_gather")
|
||||
# The kernels a read supplies, which the binding also reads; empty when it
|
||||
# supplies none.
|
||||
_READ_KERNELS = ("completing_fusions", "gathering_reads")
|
||||
|
||||
|
||||
def _require(protocol, implementation, facts):
|
||||
missing = [name for name in facts if not hasattr(implementation, name)]
|
||||
if missing:
|
||||
raise TypeError(
|
||||
f"{type(implementation).__name__} does not set {', '.join(missing)}, "
|
||||
f"which every {protocol} must"
|
||||
)
|
||||
|
||||
|
||||
def _check_update(update):
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS)
|
||||
name = type(update).__name__
|
||||
if update.writes_stream and (update.is_plain_add or not update.applied_at_exit):
|
||||
raise ValueError(
|
||||
f"{name} writes the next stream itself, which only an update "
|
||||
"applied at its exit and other than a plain add does"
|
||||
)
|
||||
if update.quantized_sum and not update.is_plain_add:
|
||||
raise ValueError(
|
||||
f"{name} lets its sum run quantized, which only a plain add does"
|
||||
)
|
||||
|
||||
|
||||
def _check_read(read):
|
||||
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -420,11 +468,13 @@ def _resolve_stage(stage, variant, following=None):
|
||||
if stage.update.applied_at_exit:
|
||||
if stage.sparse and moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
"an update applied at the stage's exit with a MoE gathered over "
|
||||
"the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_scattered_possible():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
"an update applied at the stage's exit with input-scattered "
|
||||
"attention under attention CP"
|
||||
)
|
||||
if stage.kind is StageKind.FFN:
|
||||
declaration, residual, returned = _resolve_ffn(
|
||||
@@ -439,8 +489,7 @@ def _resolve_stage(stage, variant, following=None):
|
||||
)
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION or (
|
||||
following is not None
|
||||
and getattr(following.read, "reads_after_attn_tp_gather", False)
|
||||
following is not None and following.read.reads_after_attn_tp_gather
|
||||
):
|
||||
returned = attention
|
||||
elif exit_rows is ExitRows.SLICE:
|
||||
@@ -706,11 +755,12 @@ def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
stack's first. What they build is discarded.
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
final_read: The model's final read of the stack's output, when it is
|
||||
not a plain final norm: the consumer of the last stage when no
|
||||
later layer declares one. Its ``attn_tp_gather`` gathers the rows
|
||||
that stage leaves on this rank's attention-TP slice, and with
|
||||
``reads_attn_tp_slices`` it reads that slice instead.
|
||||
final_read: The model's final read of the stack's output (a
|
||||
``FinalRead``), when it is not a plain final norm: the consumer of
|
||||
the last stage when no later layer declares one. Its
|
||||
``attn_tp_gather`` gathers the rows that stage leaves on this
|
||||
rank's attention-TP slice, and with ``reads_attn_tp_slices`` it
|
||||
reads that slice instead.
|
||||
"""
|
||||
global _stack
|
||||
outer = _stack
|
||||
|
||||
@@ -135,9 +135,8 @@ def _reduce_update_read(
|
||||
if result is not None:
|
||||
return result
|
||||
if group is SumGroup.ATTN_TP:
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = attn_tp_all_reduce(
|
||||
hidden_states, forward_batch, may_quantize=update.is_plain_add
|
||||
hidden_states, forward_batch, may_quantize=update.quantized_sum
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
@@ -31,6 +31,13 @@ class ResidualUpdate(Protocol):
|
||||
than leaving the operation for the next prepare.
|
||||
outlives_layer: Whether parameters/state remain valid after
|
||||
leaving the producer layer, including any offload or state reuse.
|
||||
writes_stream: Whether the producer computes the next stream itself
|
||||
and writes it at its exit, dropping the previous residual: what
|
||||
reaches its exit is already complete.
|
||||
quantized_sum: Whether the sum the contribution owes may be completed
|
||||
by a quantized all-reduce when quantized communication is enabled.
|
||||
This is a numerical policy, separate from being a plain add; full
|
||||
precision is always correct.
|
||||
|
||||
The shard conversion methods must move any update-associated state together
|
||||
with the residual. Nonlinear updates must not claim is_plain_add.
|
||||
@@ -46,6 +53,12 @@ class ResidualUpdate(Protocol):
|
||||
# Its parameters and state remain valid after leaving the producer layer.
|
||||
# Stateful or offloaded implementations must not opt in without that guarantee.
|
||||
outlives_layer: bool
|
||||
# The producer builds the next stream itself and writes it at its exit, so
|
||||
# its exit has no sum left to complete and the previous residual is dropped.
|
||||
# Only an update applied at the exit can, and it is not a plain add.
|
||||
writes_stream: bool
|
||||
# The sum it owes may run as a quantized all-reduce.
|
||||
quantized_sum: bool
|
||||
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""Write hidden_states, the producer contribution, into residual.
|
||||
@@ -70,17 +83,23 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: Read is normalization/optional quantization without changing
|
||||
the residual, allowing compatible fused add+norm implementations.
|
||||
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
|
||||
reads_after_attn_tp_gather: Optional. The read needs every row of the
|
||||
reads_after_attn_tp_gather: The read needs every row of the
|
||||
attention's, so it runs after any gather over attention TP: a
|
||||
producer that would leave its output on this rank's attention-TP
|
||||
slice returns it to the attention's rows instead.
|
||||
completing_fusions: Optional. ReadoutFusion kernels that complete the
|
||||
sum the input owes with the residual add ahead of this read, tried
|
||||
before the boundary's own all-reduce or reduce-scatter.
|
||||
gathering_reads: Optional. Callables ``(hidden_states, residual,
|
||||
norm)`` that read the input on this rank's attention-TP slice and
|
||||
gather it over attention TP, returning ``(input, residual)``, or
|
||||
None when they do not take the batch.
|
||||
slice returns it to the attention's rows instead. False lets it
|
||||
read that slice, its result gathered after it. Not derivable from
|
||||
the rows the stage computes on: a read on the slice followed by a
|
||||
gather and a read after the gather leave the same compute rows.
|
||||
completing_fusions: ReadoutFusion kernels that complete the sum the
|
||||
input owes with the residual add ahead of this read, tried before
|
||||
the boundary's own all-reduce or reduce-scatter. Empty when the
|
||||
read offers none: the boundary's own collective followed by
|
||||
update_and_read is correct for every read.
|
||||
gathering_reads: Callables ``(hidden_states, residual, norm)`` that
|
||||
read the input on this rank's attention-TP slice and gather it over
|
||||
attention TP, returning ``(input, residual)``, or None when they do
|
||||
not take the batch. Empty when the read offers none: it then runs
|
||||
on the slice and the boundary gathers what it read.
|
||||
|
||||
init_residual initializes the stack residual. read consumes an
|
||||
already-written residual; update_and_read first applies the actual
|
||||
@@ -95,6 +114,14 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: bool
|
||||
# Preserve the read on the source rows before a DP gather.
|
||||
reads_before_dp_gather: bool
|
||||
# The read needs every row of the attention's, not this rank's slice.
|
||||
reads_after_attn_tp_gather: bool
|
||||
# Kernels that complete the sum the input owes with the residual add
|
||||
# ahead of this read; empty when the read offers none.
|
||||
completing_fusions: Tuple
|
||||
# Reads that run on this rank's attention-TP slice and gather what they
|
||||
# read; empty when the read offers none.
|
||||
gathering_reads: Tuple
|
||||
|
||||
def init_residual(self, hidden_states) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
@@ -138,6 +165,9 @@ class ResidualReadout(Protocol):
|
||||
norm: Consumer normalization module.
|
||||
quant_format: Requested input quantization format, as in read().
|
||||
post_residual_addition: Optional extra added after update, before norm.
|
||||
Each read documents where it applies, rejects or does not
|
||||
apply it. A stream that arrives already written is read with
|
||||
``read``, which is not given it.
|
||||
|
||||
Returns:
|
||||
(compute_input, residual). A fused implementation can preserve FP32
|
||||
@@ -145,6 +175,31 @@ class ResidualReadout(Protocol):
|
||||
"""
|
||||
|
||||
|
||||
class FinalRead(Protocol):
|
||||
"""The model's read of the layer stack's output, its final norm.
|
||||
|
||||
Called with the output alone on a stream its producer already wrote, else
|
||||
with the output and the residual, returning ``(normed, residual)``: the
|
||||
residual add followed by the norm, so a pending update must be a plain add.
|
||||
|
||||
Optional, each read where it is used with the default given:
|
||||
attn_tp_gather: Its own gather over attention TP of rows the stack's
|
||||
last stage leaves on this rank's attention-TP slice, returning
|
||||
None when it does not take the batch. Without it the boundary's
|
||||
gather runs, which is always correct.
|
||||
reads_attn_tp_slices: It reads this rank's attention-TP slice of the
|
||||
rows and gathers what it read itself, so the stack's last FFN
|
||||
leaves its output on that slice. False by default: the last FFN
|
||||
returns its output on the attention's rows, which any final read
|
||||
takes.
|
||||
gemma_weight: The gamma a deferred MoE finalize normalizes with, when
|
||||
the model's final norm takes that handoff.
|
||||
"""
|
||||
|
||||
def __call__(self, hidden_states, residual=None, **read_kwargs):
|
||||
"""The final norm of ``hidden_states``, after adding ``residual``."""
|
||||
|
||||
|
||||
class LayerResidualOps(NamedTuple):
|
||||
"""The reads and updates a decoder layer declares for its attention and
|
||||
its FFN."""
|
||||
|
||||
@@ -276,6 +276,8 @@ class PlainAdd:
|
||||
is_plain_add = True
|
||||
applied_at_exit = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = True
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
@@ -297,6 +299,8 @@ class ReplaceAtExit:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = True
|
||||
writes_stream = True
|
||||
quantized_sum = False
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return hidden_states
|
||||
@@ -320,10 +324,17 @@ class NormQuantReadout:
|
||||
"""The input is the residual's norm, fused with the quantization the
|
||||
consumer asks for (``quant_format``) and a post-residual addition; a plain
|
||||
add of the previous output runs in the same kernel. An empty batch skips
|
||||
the norm."""
|
||||
the norm.
|
||||
|
||||
``post_residual_addition`` is added after a plain add and before the norm.
|
||||
It is not applied by the aiter mxfp4 and fp8-group kernels, after an
|
||||
update other than a plain add, or by ``read``, which has no add."""
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather: bool = False
|
||||
reads_after_attn_tp_gather = False
|
||||
fp8_input: Optional[Fp8Input] = None
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
@@ -355,10 +366,15 @@ class NormQuantReadout:
|
||||
@dataclass(frozen=True)
|
||||
class NormReadout:
|
||||
"""The input is the residual's norm; a plain add of the previous output
|
||||
runs in the same kernel. An empty batch skips the norm."""
|
||||
runs in the same kernel. An empty batch skips the norm. It rejects a
|
||||
``quant_format`` and a ``post_residual_addition``, except that the latter
|
||||
is not applied after an update other than a plain add."""
|
||||
|
||||
is_plain_norm = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather: bool = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
@@ -402,6 +418,8 @@ class UnfusedNormReadout(NormReadout):
|
||||
in models that add their residual themselves. No fused kernel takes it."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
|
||||
@@ -195,6 +195,8 @@ class _BankReadout:
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
|
||||
def __init__(self, state: AttnBankState):
|
||||
self.state = state
|
||||
@@ -217,7 +219,8 @@ class _BankReadout:
|
||||
|
||||
class _AttnReadout(_BankReadout):
|
||||
"""The attention input: the bank aggregation and this layer's input norm.
|
||||
A write layer snapshots the residual this read forms."""
|
||||
A write layer snapshots the residual this read forms. ``read`` rejects a
|
||||
``post_residual_addition``; ``update_and_read`` does not apply one."""
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
@@ -247,6 +250,8 @@ class _FfnReadout(_BankReadout):
|
||||
"""The FFN input: the attention output's add folded into the bank
|
||||
aggregation, and this layer's post-attention norm."""
|
||||
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
@property
|
||||
def completing_fusions(self) -> Tuple[ReadoutFusion, ...]:
|
||||
state = self.state
|
||||
|
||||
@@ -114,9 +114,12 @@ class GatedResidualState:
|
||||
|
||||
class _AttnReadout:
|
||||
"""The normalized, gated mix of the streams, from streams that already hold
|
||||
the previous layer's output: such a layer takes its input written back."""
|
||||
the previous layer's output: such a layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
@@ -150,6 +153,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
@@ -168,7 +173,10 @@ class _FfnReadout:
|
||||
"""The attention output's injection and the FFN input's mix."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
@@ -198,6 +206,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
@@ -101,10 +101,13 @@ class IHCState:
|
||||
class _AttnReadout:
|
||||
"""The gated mix of the streams and the input norm, from streams that
|
||||
already hold the previous layer's output: an iHC layer takes its input
|
||||
written back."""
|
||||
written back. A ``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -129,6 +132,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -148,7 +153,10 @@ class _FfnReadout:
|
||||
norm, fused in post_pre when the layer provides it."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
@@ -176,6 +184,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: IHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -122,10 +122,14 @@ class MHCState:
|
||||
|
||||
class _AttnReadout:
|
||||
"""hc_pre and the input norm, from streams that already hold the previous
|
||||
layer's output: an MHC layer takes its input written back."""
|
||||
layer's output: an MHC layer takes its input written back. A
|
||||
``post_residual_addition`` is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -149,6 +153,8 @@ class _AttnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -168,7 +174,10 @@ class _FfnReadout:
|
||||
in hc_ffn_post_pre when it takes the batch."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -195,6 +204,8 @@ class _FfnUpdate:
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
outlives_layer = False
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -26,6 +26,8 @@ class PostNormAdd:
|
||||
|
||||
is_plain_add = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def __init__(self, norm, *, applied_at_exit: bool = False):
|
||||
self.norm = norm
|
||||
@@ -45,7 +47,10 @@ class PlainReadout:
|
||||
"""The input is the residual itself; the stage's norm is not applied."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
@@ -35,7 +35,6 @@ from sglang.srt.layers.layer_boundary.ops import sum_output
|
||||
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
PLAIN_ADD,
|
||||
REPLACE_AT_EXIT,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.batch import stream_of
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream
|
||||
@@ -281,11 +280,11 @@ class StageBoundary:
|
||||
"""Complete the sum over ``group`` that one part of this FFN's output
|
||||
owes.
|
||||
|
||||
For an FFN that builds the next stream itself (REPLACE_AT_EXIT) from
|
||||
For an FFN that builds the next stream itself (``writes_stream``) from
|
||||
several complete parts, normalizing each before combining them; the
|
||||
stream it hands to its exit is then complete.
|
||||
"""
|
||||
if self.declaration.update is not REPLACE_AT_EXIT:
|
||||
if not self.declaration.update.writes_stream:
|
||||
raise RuntimeError(
|
||||
"only an FFN that writes the next stream itself sums its parts"
|
||||
)
|
||||
|
||||
@@ -134,7 +134,10 @@ class _ResidualMergeRead:
|
||||
dtype. The producer's update must be a plain add."""
|
||||
|
||||
is_plain_norm = False
|
||||
completing_fusions = ()
|
||||
gathering_reads = ()
|
||||
reads_before_dp_gather = False
|
||||
reads_after_attn_tp_gather = False
|
||||
|
||||
def __init__(self, res_scale: Optional[ResidualScaling]):
|
||||
self.res_scale = res_scale
|
||||
|
||||
Reference in New Issue
Block a user