[Refactor] Declare each stage's residual read and update, and give each stage its own entry (#41440)

This commit is contained in:
Cheng Wan
2026-09-27 16:55:04 -07:00
committed by GitHub
parent 0e907b755d
commit 644014bc98
22 changed files with 1059 additions and 461 deletions
@@ -25,6 +25,7 @@ from sglang.srt.layers.communicator.boundary import (
InputRead,
LayerStage,
StageDecl,
StageEntry,
StageInput,
StageOutput,
decoder_layer_edges,
@@ -67,16 +68,20 @@ from sglang.srt.layers.communicator.output import (
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.communicator.residual.add_norm import (
ADD_AND_NORM,
ADD,
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
NORM_QUANT_READ,
NORM_READ,
PLAIN_RESIDUAL,
)
from sglang.srt.layers.communicator.residual.mhc import (
MHCState,
)
__all__ = [
"ADD_AND_NORM",
"ADD",
"AttentionInputs",
"BoundarySteps",
"CommunicateContext",
@@ -92,14 +97,19 @@ __all__ = [
"HandoffOutput",
"InputRead",
"LayerCommunicator",
"LayerResidual",
"LayerScatterModes",
"LayerStage",
"Layout",
"MHCLayerCommunicator",
"MHCState",
"MixerExit",
"NORM_QUANT_READ",
"NORM_READ",
"PLAIN_RESIDUAL",
"ScatterMode",
"StageDecl",
"StageEntry",
"StageInput",
"StageOutput",
"SumGroup",
+138 -69
View File
@@ -54,8 +54,16 @@ from sglang.srt.layers.communicator.output import (
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import ResidualOps
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
from sglang.srt.layers.communicator.residual import (
LayerResidual,
StageRead,
StageUpdate,
)
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
NORM_QUANT_READ,
NORM_READ,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
@@ -68,6 +76,8 @@ class StageInput(msgspec.Struct, frozen=True):
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
# How the consumer reads its input from the residual.
read: StageRead = NORM_READ
class StageOutput(msgspec.Struct, frozen=True):
@@ -93,6 +103,8 @@ class StageOutput(msgspec.Struct, frozen=True):
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
# How the producer's output is written into the residual.
update: StageUpdate = ADD
class DecoderLayerSides(msgspec.Struct, frozen=True):
@@ -156,7 +168,14 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
construction (``input_owes``); a sum the previous layer leaves for a batch
comes with the value."""
owes = sides.input_owes is not None
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
# The previous layer's FFN output, written in as this layer's FFN writes its
# own.
arrived = StageOutput(
sides.input_rows,
group=sides.input_owes,
always_leaves=owes,
update=sides.ffn_output.update,
)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
attention_rows = (
@@ -187,6 +206,23 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
)
def with_residual(
sides: DecoderLayerSides, residual: LayerResidual
) -> DecoderLayerSides:
"""``sides`` with the reads and updates the layer declares for its
attention and its FFN."""
replace = msgspec.structs.replace
return replace(
sides,
attention=replace(sides.attention, read=residual.attention_read),
attention_output=replace(
sides.attention_output, update=residual.attention_update
),
ffn=replace(sides.ffn, read=residual.ffn_read),
ffn_output=replace(sides.ffn_output, update=residual.ffn_update),
)
def stage_edges(
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
) -> Tuple[EdgeDecl, EdgeDecl]:
@@ -198,16 +234,18 @@ def stage_edges(
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
update = previous.update if previous is not None else ADD
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
update=update,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows)
else StageOutput(rows, update=update)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
@@ -240,7 +278,7 @@ def sequence_parallel_layer_sides(
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather the slices into their GEMM.
attention=StageInput(attention, gathers_itself=gathers),
attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
# o_proj and down reduce-scatter out of theirs, whether fused or not.
attention_output=StageOutput(local),
ffn=StageInput(
@@ -281,7 +319,9 @@ def scattered_residual_layer_sides(
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
@@ -315,7 +355,9 @@ def input_scattered_layer_sides(
return DecoderLayerSides(
input_rows=attention,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
@@ -384,6 +426,7 @@ def decoder_layer_sides(
if attention_gathers_local_rows
else frozenset()
),
read=NORM_QUANT_READ,
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
@@ -475,19 +518,33 @@ def _cp_moves() -> CpMoves:
)
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into the attention,
from the attention output to the FFN input, and the FFN output on to the
rows the layer hands on."""
class StageEntry(msgspec.Struct, frozen=True):
"""The boundary into one of a layer's stages, as the layer runs it."""
# The half into the attention: completes what the input owes, writes the
# previous output into the residual and reads the attention input
# (_attention_input_step); attention_input then moves it.
attention_prepare: Callable
attention_input: Callable
ffn_input: Callable
# The rows ffn_input hands the FFN.
ffn_input_rows: Layout
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
# (hidden_states, residual, forward_batch, norm, context, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch, context) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
handoff: Optional[Callable] = None
# The fused kernels prepare tries first.
fused: Tuple["FusedMlpInput", ...] = ()
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into each of its
stages, and the last stage's output on to the rows the layer hands on."""
# The boundary into the layer's attention and into its FFN; None for a
# stage a layer that is one stage does not have.
attention: Optional[StageEntry]
ffn: Optional[StageEntry]
# What the FFN exit reads: the FFN output's group and what it may leave.
ffn_output: StageOutput
# The postprocess that moves the FFN output on; None when it goes back over
@@ -497,11 +554,6 @@ class BoundarySteps(msgspec.Struct, frozen=True):
ffn_sum_is_movable: bool
# Whether ffn_output_move also completes the sum the FFN leaves.
ffn_output_move_completes_sum: bool = False
# The fused kernels ffn_input tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Hands the attention its input once attention_input has moved it:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
attention_handoff: Callable = _hand_qkv_hook_its_input
@property
def returns_over_dp(self) -> bool:
@@ -520,7 +572,8 @@ def _attention_input_step(
layer_input: Optional[Callable],
fusions: Tuple[Callable, ...],
enters_stack: bool,
residual_ops: ResidualOps,
read: StageRead,
update: StageUpdate,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""A boundary's half into an attention: complete what the previous layer
left (a value that owes a sum, or a producer's handoff one of ``fusions``
@@ -549,22 +602,17 @@ def _attention_input_step(
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual, context)
if enters:
hidden_states, residual = residual_ops.enter(hidden_states), None
hidden_states, residual = read.enter(hidden_states), None
if owed is not None and hidden_states.shape[0] != 0:
hidden_states = reduce_output(owed)
if residual is None:
# The previous layer already wrote its output into the residual.
return residual_ops.read_attention_input(hidden_states, norm, quant_format)
return residual_ops.update_and_read_attention_input(
hidden_states, residual, norm, quant_format, post_residual_addition
return read.read(hidden_states, norm, quant_format)
return read.update_and_read(
update, hidden_states, residual, norm, quant_format, post_residual_addition
)
def _another_stage(*args, **kwargs):
"""The steps of a stage a single-stage layer does not have."""
raise RuntimeError("this layer is one stage and does not have the other")
class InputRead(Enum):
"""How a boundary's consumer reads its input from the residual: with the
attention input norm (prepare_attn) or with the FFN input norm and its
@@ -610,11 +658,10 @@ class Boundary(msgspec.Struct, frozen=True):
@property
def input_rows(self) -> Layout:
"""The rows the consumer is handed: what it needs, still sharded over
the axes it gathers itself."""
the axes it gathers itself as the rows its input is read on are."""
need = self.edge.need
return Layout(
need.layout.sharded
| (self.edge.produced.layout.sharded & need.gathers_itself)
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
)
@@ -625,7 +672,6 @@ def make_boundary(
fusions: Tuple = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
enters_stack: bool = False,
) -> Boundary:
"""The steps a layer runs at ``edge``, around the residual operations;
@@ -646,7 +692,7 @@ def make_boundary(
residual=edge.residual,
to=edge.residual_to,
cp_moves=cp_moves,
residual_ops=residual_ops,
update=edge.produced.update,
)
return Boundary(
edge,
@@ -663,7 +709,8 @@ def make_boundary(
fusions=fusions,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
residual_ops=residual_ops,
read=edge.need.read,
update=edge.produced.update,
)
return Boundary(edge, prepare=input_step, fused=fused)
layer_input = None
@@ -683,9 +730,15 @@ def make_boundary(
prepare=partial(
_attention_input_step,
layer_input=layer_input,
fusions=fusions,
# A fused kernel runs the add and the norm itself.
fusions=(
fusions
if edge.produced.update.adds_plainly and edge.need.read.norms_plainly
else ()
),
enters_stack=enters_stack,
residual_ops=residual_ops,
read=edge.need.read,
update=edge.produced.update,
),
input_move=_select_attention_input_move(edge.residual_to, edge.need),
)
@@ -697,7 +750,6 @@ def _select_boundary_steps(
fusions: Tuple["FusedMlpInput", ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
attention_handoff: Callable = _hand_qkv_hook_its_input,
attention_fusions: Tuple[Callable, ...] = (),
enters_stack: bool = False,
@@ -708,35 +760,36 @@ def _select_boundary_steps(
input tries ``attention_fusions``, and the layer stack's first layer
(``enters_stack``) starts its residual there."""
edges = decoder_layer_edges(sides)
out_of_ffn = make_boundary(
edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops
)
out_of_ffn = make_boundary(edges.out_of_ffn, reads=None, cp_moves=cp_moves)
into_ffn = make_boundary(
edges.into_ffn,
reads=InputRead.FFN,
fusions=fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
into_attention = make_boundary(
edges.into_attention,
reads=InputRead.ATTENTION,
fusions=attention_fusions,
residual_ops=residual_ops,
enters_stack=enters_stack,
)
return BoundarySteps(
attention_prepare=into_attention.prepare,
attention_input=into_attention.input_move,
ffn_input=into_ffn.prepare,
ffn_input_rows=into_ffn.input_rows,
attention=StageEntry(
prepare=into_attention.prepare,
input_rows=into_attention.input_rows,
input_move=into_attention.input_move,
handoff=attention_handoff,
),
ffn=StageEntry(
prepare=into_ffn.prepare,
input_rows=into_ffn.input_rows,
fused=into_ffn.fused,
),
ffn_output=edges.out_of_ffn.produced,
ffn_output_move=out_of_ffn.output_move,
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
fused=into_ffn.fused,
attention_handoff=attention_handoff,
)
@@ -765,7 +818,8 @@ def _select_ffn_input(
fusions: Tuple[FusedMlpInput, ...],
residual_joins_sum: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
"""The steps from the attention output to the FFN input, and the fused
kernels they try first: complete the attention-TP sum, move the residual to
@@ -791,7 +845,8 @@ def _select_ffn_input(
fusions=fusions,
residual_joins_sum=residual_joins_sum,
cp_moves=cp_moves,
residual_ops=residual_ops,
read=read,
update=update,
)
return partial(_mlp_input_completing_owed, step=step), fused
# What the attention output owes decides the steps: the attention-TP sum,
@@ -819,7 +874,8 @@ def _select_ffn_input(
partial(
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
scatters_residual=residual != residual_to,
residual_ops=residual_ops,
read=read,
update=update,
),
(),
)
@@ -828,7 +884,10 @@ def _select_ffn_input(
# attention's rows (MHC on an input-scattered batch).
if gathered or not owes_attention_tp or residual != residual_to:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
return (
partial(_mlp_input_on_residual_shard, read=read, update=update),
(),
)
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the CP moves gather them.
if cp_moves is None:
@@ -841,7 +900,8 @@ def _select_ffn_input(
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=residual_joins_sum,
residual_ops=residual_ops,
read=read,
update=update,
)
return partial(cp_moves.gather, gather=on_chunk), fused
if (
@@ -860,20 +920,26 @@ def _select_ffn_input(
gathers_residual = residual != residual_to
if not gathered:
if not owes_attention_tp:
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
return partial(_mlp_input_norm, read=read, update=update), ()
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
if not residual_ops.adds_plainly:
if not update.adds_plainly:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return _mlp_input_residual_into_sum, ()
fused = tuple(f for f in fusions if f.completes is produced.group)
return partial(_mlp_input_residual_into_sum, read=read), ()
# A fused kernel runs the add and the norm itself.
fused = (
tuple(f for f in fusions if f.completes is produced.group)
if update.adds_plainly and read.norms_plainly
else ()
)
return (
partial(
_mlp_input_without_dp,
gathers_residual=gathers_residual,
fusions=tuple(f.run for f in fused),
residual_ops=residual_ops,
read=read,
update=update,
),
fused,
)
@@ -886,13 +952,15 @@ def _select_ffn_input(
if (
owes_attention_tp
and not force_layernorm_before_gather
and residual_ops.adds_plainly
and update.adds_plainly
and read.norms_plainly
):
return (
partial(
_mlp_input_dp_partial,
gathers_residual=gathers_residual,
places_cp_shards=places_cp_shards,
read=read,
),
(),
)
@@ -902,7 +970,8 @@ def _select_ffn_input(
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
places_cp_shards=places_cp_shards,
residual_ops=residual_ops,
read=read,
update=update,
),
(),
)
@@ -914,7 +983,7 @@ def _select_ffn_output_move(
residual: Layout,
to: Layout,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
update: StageUpdate = ADD,
) -> Tuple[bool, Optional[Callable], bool]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
@@ -928,7 +997,7 @@ def _select_ffn_output_move(
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: write the output
# into the residual, then gather over attention TP.
return False, partial(pair._gather, residual_ops=residual_ops), False
return False, partial(pair._gather, update=update), False
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
@@ -942,7 +1011,7 @@ def _select_ffn_output_move(
pair._onto_residual_shard,
sums=sums,
gathers_back=to != residual,
residual_ops=residual_ops,
update=update,
),
sums,
)
+113 -69
View File
@@ -15,7 +15,7 @@
from dataclasses import dataclass
from enum import Enum, auto
from functools import partial
from functools import cached_property, partial
from typing import Callable, Optional, Tuple, Union
import msgspec
@@ -33,7 +33,7 @@ from sglang.srt.layers.communicator.boundary import (
FusedMlpInput,
InputRead,
LayerStage,
_another_stage,
StageEntry,
_cp_moves,
_select_boundary_steps,
decoder_layer_sides,
@@ -41,6 +41,7 @@ from sglang.srt.layers.communicator.boundary import (
make_boundary,
scattered_residual_layer_sides,
sequence_parallel_layer_sides,
with_residual,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
@@ -73,9 +74,9 @@ from sglang.srt.layers.communicator.output import (
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import ResidualOps
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.communicator.residual.add_norm import (
ADD_AND_NORM,
PLAIN_RESIDUAL,
aiter_all_reduce_fusion_enabled_for,
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
@@ -244,13 +245,68 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce()
class StageCommunicator:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
self._layer = layer
self._name = name
self._norm_name = norm_name
@property
def norm(self) -> torch.nn.Module:
"""The norm the stage reads its input with, one of the layer's."""
return getattr(self._layer, self._norm_name)
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
) -> StageEntry:
if steps is None:
steps = self._layer._batch_steps(forward_batch)
entry = getattr(steps, self._name)
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self._name} stage"
)
return entry
def prepare(
self,
hidden_states,
residual,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
context = self._layer._context
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, context, **call
)
if entry.input_move is not None:
hidden_states = entry.input_move(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=context,
)
if entry.handoff is not None:
hidden_states = entry.handoff(
hidden_states, forward_batch, self._layer.qkv_latent_func
)
return hidden_states, residual
class LayerCommunicator:
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
# layout and try no fused kernel at the FFN exit.
_publish_lora_layout: bool = False
_ffn_exit_fusions: Tuple[Callable, ...] = ()
# A plain residual unless the layer is built with its own.
_residual_ops: ResidualOps = ADD_AND_NORM
_residual: LayerResidual = PLAIN_RESIDUAL
def __init__(
self,
@@ -265,8 +321,9 @@ class LayerCommunicator:
fused_ar_quant_keep_bf16: bool = False,
# False for a layer whose FFN always completes its own reduction.
allow_deferred_ffn_reduction: bool = True,
# How the layer writes its residual and reads its stages' inputs.
residual_ops: ResidualOps = ADD_AND_NORM,
# How the layer's attention and FFN read their inputs from the residual
# and write their outputs into it.
residual: LayerResidual = PLAIN_RESIDUAL,
# A layer that is one stage of a sequence of stages, instead of an
# attention followed by an FFN.
stage: Optional["LayerStage"] = None,
@@ -281,7 +338,7 @@ class LayerCommunicator:
self.enable_fused_ar_quant = enable_fused_ar_quant
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
self._residual_ops = residual_ops
self._residual = residual
self._context = CommunicateContext.init_new()
self._context.force_layernorm_before_dp_gather = (
@@ -299,6 +356,11 @@ class LayerCommunicator:
get_lora().enable_lora
)
if stage is not None:
if residual != PLAIN_RESIDUAL:
raise ValueError(
"a layer that is one stage takes its reads and updates from "
"the stage's declarations"
)
self._init_stage(stage)
return
# Its two boundaries, for a layer that is one stage.
@@ -336,8 +398,10 @@ class LayerCommunicator:
# q_lora, which input-scattered attention needs.
self._sp_steps = (
_select_boundary_steps(
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
residual_ops=residual_ops,
with_residual(
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
residual,
),
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
)
@@ -354,9 +418,9 @@ class LayerCommunicator:
)
self._declared = None
self._cp_steps = self._input_scattered_steps = self._sp_steps = None
self.stage_edges = stage.edges
into_edge, out_edge = stage.edges
reads_ffn = stage.reads is InputRead.FFN
self.stage_edges = stage.edges
into = make_boundary(
into_edge,
reads=stage.reads,
@@ -366,22 +430,35 @@ class LayerCommunicator:
else self._attn_input_fusions
),
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
residual_ops=self._residual_ops,
enters_stack=stage.enters_stack,
)
out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops)
out = make_boundary(out_edge, reads=None)
entry = StageEntry(
prepare=into.prepare,
input_rows=into.input_rows,
input_move=None if reads_ffn else into.input_move,
handoff=None if reads_ffn else _hand_qkv_hook_its_input,
fused=into.fused,
)
self._steps = BoundarySteps(
attention_prepare=_another_stage if reads_ffn else into.prepare,
attention_input=_another_stage if reads_ffn else into.input_move,
ffn_input=into.prepare if reads_ffn else _another_stage,
ffn_input_rows=into.input_rows,
attention=None if reads_ffn else entry,
ffn=entry if reads_ffn else None,
ffn_output=out_edge.produced,
ffn_output_move=out.output_move,
ffn_output_move_completes_sum=out.output_move_completes_sum,
ffn_sum_is_movable=out_edge.produced.group is not None,
fused=into.fused,
)
@cached_property
def attn(self) -> StageCommunicator:
"""The layer's attention stage."""
return StageCommunicator(self, "attention", "input_layernorm")
@cached_property
def ffn(self) -> StageCommunicator:
"""The layer's FFN stage."""
return StageCommunicator(self, "ffn", "post_attention_layernorm")
@property
def input_rows(self) -> Layout:
"""The rows the layer's input and residual arrive on in a batch that
@@ -472,7 +549,7 @@ class LayerCommunicator:
# No batch shards its tokens and there is no attention DP: as
# without CP.
may_leave = may_leave_to_reduce_scatter = True
return decoder_layer_sides(
sides = decoder_layer_sides(
axis_sizes=token_axis_sizes(cp_active=cp_active),
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
ffn_shards_over_cp=ffn_on_cp_shards,
@@ -499,6 +576,7 @@ class LayerCommunicator:
)
and may_leave,
)
return with_residual(sides, self._residual)
@staticmethod
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
@@ -532,7 +610,7 @@ class LayerCommunicator:
for the layer's ordinary declarations ``sides``. A plain residual comes
back to the full rows inside the attention output's sum; one whose
write-back is not a plain add stays on each rank's slice."""
if self._residual_ops.adds_plainly:
if self._residual.attention_update.adds_plainly:
scattered = input_scattered_layer_sides(
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
@@ -550,8 +628,7 @@ class LayerCommunicator:
# DSA and hook-less attention take the slice gathered.
handoff = _hand_scattered_input_to_attention
return _select_boundary_steps(
scattered,
residual_ops=self._residual_ops,
with_residual(scattered, self._residual),
attention_handoff=handoff,
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
@@ -563,7 +640,6 @@ class LayerCommunicator:
"""The steps this layer runs for a set of declarations."""
return _select_boundary_steps(
sides,
residual_ops=self._residual_ops,
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
**kwargs,
@@ -586,9 +662,7 @@ class LayerCommunicator:
post_residual_addition=post_residual_addition,
)
if captured_last_layer_outputs is not None:
gathered_last_layer_output = self._batch_steps(
forward_batch
).attention_input(
gathered_last_layer_output = self.attn.entry(forward_batch).input_move(
hidden_states=residual,
forward_batch=forward_batch,
context=self._context,
@@ -616,10 +690,7 @@ class LayerCommunicator:
the batch is not input-scattered.
"""
return (
any(
f.may_return_new_residual
for f in self._batch_steps(forward_batch).fused
)
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
@@ -637,7 +708,7 @@ class LayerCommunicator:
"lora_batch_layout",
(
LoRABatchLayout.DP_LOCAL
if TokenAxis.ATTN_DP in steps.ffn_input_rows.sharded
if TokenAxis.ATTN_DP in steps.ffn.input_rows.sharded
else LoRABatchLayout.TP_GLOBAL
),
)
@@ -660,20 +731,13 @@ class LayerCommunicator:
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
hidden_states, residual = self._batch_steps(forward_batch).attention_prepare(
return self.attn.prepare(
hidden_states,
residual,
forward_batch,
self.input_layernorm,
self._context,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
return self._finish_prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
def _in_sp_region(self) -> bool:
return self._sp_steps is not None and get_forward().sp_active
@@ -693,27 +757,14 @@ class LayerCommunicator:
return self._cp_steps
return self._steps
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
steps = self._batch_steps(forward_batch)
hidden_states = steps.attention_input(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
hidden_states = steps.attention_handoff(
hidden_states, forward_batch, self.qkv_latent_func
)
return hidden_states, residual
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
"""The fused kernels that complete what the previous layer left together
with the residual update and the input norm, in the order they are tried.
Each takes (owed, residual, forward_batch, post_residual_addition) and
returns None when it does not take the batch. They add the residual
plainly."""
plainly: what arrives is written in as this layer's FFN writes its own."""
if not (
self._residual_ops.adds_plainly
self._residual.ffn_update.adds_plainly
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
):
return ()
@@ -772,7 +823,7 @@ class LayerCommunicator:
"""The fused kernels that can take the attention -> FFN steps, in the
order they are tried. They add the residual plainly."""
if not (
self._residual_ops.adds_plainly
self._residual.attention_update.adds_plainly
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
):
return ()
@@ -813,13 +864,7 @@ class LayerCommunicator:
if cache is not None:
self._context.cache = cache
return steps.ffn_input(
hidden_states,
residual,
forward_batch,
self.post_attention_layernorm,
self._context,
)
return self.ffn.prepare(hidden_states, residual, forward_batch, steps)
def maybe_prefetch_next_full_attention_kv(
self,
@@ -847,10 +892,8 @@ class LayerCommunicator:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, when
the next layer's input also writes the output into the residual."""
return (
self._batch_steps(forward_batch).returns_over_dp
and not self._residual_ops.updates_residual_after_ffn
)
steps = self._batch_steps(forward_batch)
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
@@ -977,8 +1020,9 @@ class LayerCommunicator:
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
if residual is not None and self._residual_ops.updates_residual_after_ffn:
hidden_states = self._residual_ops.update_residual(hidden_states, residual)
update = steps.ffn_output.update
if residual is not None and update.at_producer:
hidden_states = update.update(hidden_states, residual)
residual = None
return hidden_states, residual
@@ -1326,7 +1370,7 @@ class MHCLayerCommunicator(LayerCommunicator):
allow_reduce_scatter,
qkv_latent_func,
allow_deferred_ffn_reduction=False,
residual_ops=self.mhc,
residual=self.mhc.layer_residual(),
)
# MHC has not been run with input-scattered attention under attention CP.
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
+42 -34
View File
@@ -41,8 +41,8 @@ from sglang.srt.layers.communicator.layout import (
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output
from sglang.srt.layers.communicator.residual import ResidualOps
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.communicator_dsa_cp import (
dsa_cp_gather_hidden_states,
dsa_cp_reduce_scatter_hidden_states,
@@ -249,21 +249,22 @@ def _mlp_input_without_dp(
*,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
if gathers_residual:
residual = residual_ops.residual_from_attn_tp_shards(residual)
residual = update.residual_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
# MHC sums its streams in full precision.
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
hidden_states, forward_batch, may_quantize=update.adds_plainly
)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_dp_replicate(
@@ -276,14 +277,15 @@ def _mlp_input_dp_replicate(
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: complete the attention-TP sum if it is owed, write the
output into the residual and read the FFN input locally, then gather. With
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
beside the others' in the group's slot."""
if gathers_residual:
residual = residual_ops.residual_from_attn_tp_shards(residual)
residual = update.residual_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
@@ -291,12 +293,12 @@ def _mlp_input_dp_replicate(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
else:
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
return (
@@ -314,10 +316,11 @@ def _mlp_input_dp_partial(
*,
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
):
"""Attention DP: one rank adds the residual, the gather sums it, then
normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP
group's tokens beside the others' in the group's slot."""
"""Attention DP: one rank adds the residual, the gather sums it, then read
the input from the sum. With ``places_cp_shards`` each CP rank puts its
shard of the DP group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if context.attn_tp_rank == 0:
@@ -327,8 +330,7 @@ def _mlp_input_dp_partial(
hidden_states, forward_batch, cp_shard_counts
)
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
if hidden_states.shape[0] != 0:
hidden_states = layernorm(hidden_states)
hidden_states, _ = read.read(hidden_states, layernorm)
return hidden_states, residual
@@ -402,9 +404,10 @@ def _mlp_input_norm(
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_scatter(
@@ -415,14 +418,15 @@ def _mlp_input_scatter(
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
if scatters_residual:
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
residual = update.residual_to_attn_tp_shard(residual, context)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_slice(
@@ -433,12 +437,13 @@ def _mlp_input_slice(
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
if scatters_residual and residual is not None:
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
residual = update.residual_to_attn_tp_shard(residual, context)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
@@ -448,7 +453,8 @@ def _mlp_input_on_residual_shard(
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps,
read: StageRead,
update: StageUpdate,
):
"""The residual stays on each rank's slice while the FFN takes the full
rows: reduce-scatter the attention output onto the slice, which completes
@@ -458,7 +464,7 @@ def _mlp_input_on_residual_shard(
return hidden_states, hidden_states
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
shard, residual = read.update_and_read(update, shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
@@ -469,9 +475,11 @@ def _mlp_input_residual_into_sum(
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
read: StageRead = NORM_READ,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context
hidden_states, residual, layernorm, context, read
)
@@ -480,6 +488,7 @@ def _tp_all_reduce_with_scattered_residual(
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
read: StageRead = NORM_READ,
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
@@ -487,8 +496,7 @@ def _tp_all_reduce_with_scattered_residual(
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = layernorm(residual)
return hidden_states, residual
return read.read(residual, layernorm)
def _mlp_input_gather_attention_cp(
@@ -669,10 +677,10 @@ class CommunicateSummableTensorPairFn:
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
residual_ops: ResidualOps = ADD_AND_NORM,
update: StageUpdate = ADD,
**kwargs,
):
hidden_states = residual_ops.update_residual(hidden_states, residual)
hidden_states = update.update(hidden_states, residual)
return _redistribute_from_attn_tp_shards(hidden_states), None
@staticmethod
@@ -684,7 +692,7 @@ class CommunicateSummableTensorPairFn:
*,
sums: bool,
gathers_back: bool,
residual_ops: ResidualOps,
update: StageUpdate,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
@@ -697,7 +705,7 @@ class CommunicateSummableTensorPairFn:
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
if not gathers_back:
return hidden_states, residual
local_states = residual_ops.update_residual(hidden_states, residual)
local_states = update.update(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
)
@@ -11,50 +11,77 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""How a layer writes its stages' outputs into the residual and reads the next
stage's input back."""
"""How a stage writes its output into the residual (its update) and how a
stage reads its input back from it (its read). The producer of a boundary
declares the update, the consumer the read; the boundary steps complete sums
and move tokens around the two."""
from typing import Protocol, Tuple
from typing import NamedTuple, Optional, Protocol, Tuple
import torch
class ResidualOps(Protocol):
"""How a layer writes each stage's output into its residual and reads the
next stage's input from it. The boundary steps complete sums and move
tokens around these operations. AddAndNorm adds and normalizes; MHC's
hyper-connection streams implement the same operations (residual.mhc)."""
class StageUpdate(Protocol):
"""How a stage's output is written into the residual."""
# The write-back is a plain add, which one rank may run before the sum it
# adds into completes: the DP partial order, and a residual that joins the
# attention output's sum.
# A plain add, which one rank may run before the sum it adds into
# completes (the DP partial order, a residual that joins the attention
# output's sum), and which a fused add + norm kernel may run.
adds_plainly: bool
# The layer writes its FFN output into the residual itself instead of
# leaving that to the next layer's input.
updates_residual_after_ffn: bool
# The stage writes its output into the residual itself at its end, instead
# of leaving that to the next stage's read.
at_producer: bool
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
"""The attention input and the residual, from a residual that already
holds the previous layer's output."""
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
) -> Tuple:
"""Write the previous layer's output into the residual, then read the
attention input from it."""
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
"""Write the attention output into the residual, then read the FFN
input from it."""
def update_residual(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the FFN output written into it."""
def update(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the output written into it."""
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
"""This attention-TP rank's slice of the residual."""
"""This attention-TP rank's slice of the residual, with whatever the
update reads along with it."""
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
"""The residual gathered from every attention-TP rank's slice."""
class StageRead(Protocol):
"""How a stage reads its input from the residual."""
# The input is the residual's norm, in the quantization the call asks for,
# and the residual is left as it is: what a fused add + norm kernel computes,
# and what may run on the rows a sum completes onto.
norms_plainly: bool
def enter(self, hidden_states) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
def read(
self,
residual,
norm,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple:
"""The input and the residual, from a residual that already holds the
previous stage's output."""
def update_and_read(
self,
update: StageUpdate,
hidden_states,
residual,
norm,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple:
"""Write the previous stage's output into the residual with its
producer's ``update``, then read the input."""
class LayerResidual(NamedTuple):
"""The reads and updates a decoder layer declares for its attention and
its FFN."""
attention_read: StageRead
attention_update: StageUpdate
ffn_read: StageRead
ffn_update: StageUpdate
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.adapters.attention import (
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.flashinfer_comm_fusion import (
is_flashinfer_allreduce_unavailable,
@@ -261,43 +262,13 @@ def _attn_input_update_and_read_residual(quant_format: str):
return _update_and_read_residual_plain
class AddAndNorm:
"""A plain residual: a stage's output is added into it, and the next
stage's input is its norm. An empty batch skips the norms."""
class Add:
"""A plain residual: the output is added into it."""
adds_plainly = True
updates_residual_after_ffn = False
at_producer = False
def enter(self, hidden_states):
return hidden_states
def read_attention_input(self, residual, norm, quant_format):
if residual.shape[0] == 0:
return residual, residual
return _attn_input_update_and_read_residual(quant_format)(
norm, residual, None, None
)
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format, post_residual_addition
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return _attn_input_update_and_read_residual(quant_format)(
norm, hidden_states, residual, post_residual_addition
)
def update_and_read_ffn_input(self, hidden_states, residual, norm):
if residual is None:
# The layer stack starts at this FFN: its input is the residual.
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return norm(hidden_states), hidden_states
if hidden_states.shape[0] == 0:
return hidden_states, residual
return norm(hidden_states, residual)
def update_residual(self, hidden_states, residual):
def update(self, hidden_states, residual):
hidden_states += residual
return hidden_states
@@ -308,4 +279,91 @@ class AddAndNorm:
return _redistribute_from_attn_tp_shards(residual)
ADD_AND_NORM = AddAndNorm()
class NormQuantRead:
"""The input is the residual's norm, fused with the quantization the
consumer asks for (``quant_format``) and a post-residual addition; a plain
add of the previous output runs in the same kernel. An empty batch skips
the norm."""
norms_plainly = True
def enter(self, hidden_states):
return hidden_states
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if residual.shape[0] == 0:
return residual, residual
return _attn_input_update_and_read_residual(quant_format)(
norm, residual, None, None
)
def update_and_read(
self,
update,
hidden_states,
residual,
norm,
quant_format="",
post_residual_addition=None,
):
if not update.adds_plainly:
return self.read(update.update(hidden_states, residual), norm, quant_format)
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return _attn_input_update_and_read_residual(quant_format)(
norm, hidden_states, residual, post_residual_addition
)
class NormRead:
"""The input is the residual's norm; a plain add of the previous output
runs in the same kernel. An empty batch skips the norm."""
norms_plainly = True
def enter(self, hidden_states):
return hidden_states
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if quant_format or post_residual_addition is not None:
raise NotImplementedError(
f"a norm read with {quant_format=} or a post-residual addition"
)
if residual.shape[0] == 0:
return residual, residual
return norm(residual), residual
def update_and_read(
self,
update,
hidden_states,
residual,
norm,
quant_format="",
post_residual_addition=None,
):
if residual is None:
# The layer stack starts at this stage: its input is the residual.
return self.read(hidden_states, norm, quant_format, post_residual_addition)
if not update.adds_plainly:
return self.read(update.update(hidden_states, residual), norm, quant_format)
if quant_format or post_residual_addition is not None:
raise NotImplementedError(
f"a norm read with {quant_format=} or a post-residual addition"
)
if hidden_states.shape[0] == 0:
return hidden_states, residual
return norm(hidden_states, residual)
ADD = Add()
NORM_QUANT_READ = NormQuantRead()
NORM_READ = NormRead()
# A plain residual: the attention reads with its input norm and the quantization
# it wants, the FFN with its norm, and each stage's output is added.
PLAIN_RESIDUAL = LayerResidual(
attention_read=NORM_QUANT_READ,
attention_update=ADD,
ffn_read=NORM_READ,
ffn_update=ADD,
)
@@ -19,17 +19,18 @@ from typing import Callable, Optional
import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.communicator.residual import LayerResidual
@dataclass
class MHCState:
"""A layer's residual as hyper-connection streams, with the residual
operations the boundary steps run (communicator.ResidualOps): the residual
is hc_mult streams, a stage's output is written in with hc_post and the
next stage's input read with hc_pre and the norm. A read produces the
h_res / h_post the next write-back consumes; they move with the tokens.
Parameters belong to the owning layer; this state only holds scratch
shared across communication stages."""
"""A layer's residual as hyper-connection streams: the residual is hc_mult
streams, a stage's output is written in with hc_post and the next stage's
input read with hc_pre and the norm. A read produces the h_res / h_post the
next write-back consumes; they move with the tokens. ``layer_residual()``
gives the reads and updates the layer's two stages declare. Parameters
belong to the owning layer; this state only holds scratch shared across
communication stages."""
hc_mult: int
hc_attn_pre: Callable
@@ -42,13 +43,6 @@ class MHCState:
h_res: Optional[torch.Tensor] = None
h_post: Optional[torch.Tensor] = None
# hc_post is not a plain add, so it runs only once the sum it writes in
# is complete.
adds_plainly = False
# The FFN output's write-back uses the coefficients this layer's FFN input
# read produced, so the layer runs it itself.
updates_residual_after_ffn = True
@staticmethod
def _resolve_out_norm(out_norm):
if out_norm is None:
@@ -102,30 +96,6 @@ class MHCState:
self.h_res = None
self.h_post = None
def enter(self, hidden_states):
return hc_expand(hidden_states, self.hc_mult)
def read_attention_input(self, residual, norm, quant_format):
if quant_format:
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
return self.attn_split(residual, out_norm=norm)
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format, post_residual_addition
):
# An MHC layer's input arrives with the previous FFN output written in.
raise NotImplementedError("an MHC layer takes its input written back")
def update_and_read_ffn_input(self, hidden_states, residual, norm):
return self.attn_to_mlp(hidden_states, residual, out_norm=norm)
def update_residual(self, hidden_states, residual):
hidden_states = self.mlp_combine(hidden_states, residual)
self.reset_aux()
if self.is_last_layer:
hidden_states = hc_contract(hidden_states, self.hc_mult)
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
rank, size = context.attn_tp_rank, context.attn_tp_size
self.h_res = self.h_res.tensor_split(size)[rank]
@@ -136,3 +106,98 @@ class MHCState:
raise NotImplementedError(
"Unsupported: h_res/h_post allgather not implemented."
)
def layer_residual(self) -> LayerResidual:
return LayerResidual(
attention_read=_AttentionRead(self),
attention_update=_AttentionUpdate(self),
ffn_read=_FfnRead(self),
ffn_update=_FfnUpdate(self),
)
class _AttentionRead:
"""hc_pre and the input norm, from streams that already hold the previous
layer's output: an MHC layer takes its input written back."""
norms_plainly = False
def __init__(self, state: MHCState):
self.state = state
def enter(self, hidden_states):
return hc_expand(hidden_states, self.state.hc_mult)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
if quant_format:
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
return self.state.attn_split(residual, out_norm=norm)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
raise NotImplementedError("an MHC layer takes its input written back")
class _AttentionUpdate:
"""hc_post with the coefficients the attention's read produced. It is not a
plain add, so it runs only once the sum it writes in is complete."""
adds_plainly = False
at_producer = False
def __init__(self, state: MHCState):
self.state = state
def update(self, hidden_states, residual):
return self.state.mlp_combine(hidden_states, residual)
def residual_to_attn_tp_shard(self, residual, context):
return self.state.residual_to_attn_tp_shard(residual, context)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
class _FfnRead:
"""The attention output's hc_post and the FFN input's hc_pre and norm, fused
in hc_ffn_post_pre when it takes the batch."""
norms_plainly = False
def __init__(self, state: MHCState):
self.state = state
def enter(self, hidden_states):
return hc_expand(hidden_states, self.state.hc_mult)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
raise NotImplementedError("an MHC FFN input read without its attention")
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
if not isinstance(update, _AttentionUpdate) or update.state is not self.state:
raise NotImplementedError(f"an MHC FFN input after {update=}")
return self.state.attn_to_mlp(hidden_states, residual, out_norm=norm)
class _FfnUpdate:
"""hc_post with the coefficients the FFN input's read produced, which this
layer runs itself; the last layer also contracts the streams into the
hidden states the layer stack hands on."""
adds_plainly = False
at_producer = True
def __init__(self, state: MHCState):
self.state = state
def update(self, hidden_states, residual):
hidden_states = self.state.mlp_combine(hidden_states, residual)
self.state.reset_aux()
if self.state.is_last_layer:
hidden_states = hc_contract(hidden_states, self.state.hc_mult)
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
return self.state.residual_to_attn_tp_shard(residual, context)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
@@ -361,7 +361,7 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
and self._context.tp_size > 1
# The FFN runs on the full rows, not each rank's own slice.
and TokenAxis.ATTN_TP_SCATTER
not in self._batch_steps(forward_batch).ffn_input_rows.sharded
not in self._batch_steps(forward_batch).ffn.input_rows.sharded
)
+8 -6
View File
@@ -4,6 +4,7 @@ from typing import Optional
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
InputRead,
LayerCommunicator,
LayerScatterModes,
@@ -37,11 +38,12 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]:
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
"""What the stage at ``layer_idx`` declares. A mixer computes on the
attention's rows and leaves its attention-TP sum to an FFN stage after it,
or, when a fused kernel takes it, to a mixer after it. An FFN computes on
the TP group's rows, a MoE dispatched by an a2a backend on this rank's own,
and may leave its sum to a mixer after it."""
"""What the stage at ``layer_idx`` declares. A mixer reads its input as
an attention does, computes on the attention's rows and leaves its
attention-TP sum to an FFN stage after it, or, when a fused kernel takes
it, to a mixer after it. An FFN computes on the TP group's rows, a MoE
dispatched by an a2a backend on this rank's own, and may leave its sum to a
mixer after it."""
axis_sizes = token_axis_sizes()
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
@@ -50,7 +52,7 @@ def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
if is_attn_layer(pattern[layer_idx]):
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return StageDecl(
StageInput(attention),
StageInput(attention, read=NORM_QUANT_READ),
StageOutput(
attention,
group=SumGroup.ATTN_TP if owes else None,
+2 -2
View File
@@ -10,7 +10,7 @@ from unittest import mock
from sglang.srt.layers import communicator
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
from sglang.srt.layers.communicator import residual as residual_ops
from sglang.srt.layers.communicator import residual as residual_contract
from sglang.srt.layers.communicator.adapters import attention
from sglang.srt.layers.communicator.residual import add_norm, mhc
@@ -19,7 +19,7 @@ COMMUNICATOR_MODULES = (
layout,
output,
attention,
residual_ops,
residual_contract,
add_norm,
mhc,
ops,