[Refactor] Declare each stage's residual read and update, and give each stage its own entry (#41440)
This commit is contained in:
@@ -25,6 +25,7 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
InputRead,
|
||||
LayerStage,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
decoder_layer_edges,
|
||||
@@ -67,16 +68,20 @@ from sglang.srt.layers.communicator.output import (
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD_AND_NORM,
|
||||
ADD,
|
||||
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
PLAIN_RESIDUAL,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import (
|
||||
MHCState,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ADD_AND_NORM",
|
||||
"ADD",
|
||||
"AttentionInputs",
|
||||
"BoundarySteps",
|
||||
"CommunicateContext",
|
||||
@@ -92,14 +97,19 @@ __all__ = [
|
||||
"HandoffOutput",
|
||||
"InputRead",
|
||||
"LayerCommunicator",
|
||||
"LayerResidual",
|
||||
"LayerScatterModes",
|
||||
"LayerStage",
|
||||
"Layout",
|
||||
"MHCLayerCommunicator",
|
||||
"MHCState",
|
||||
"MixerExit",
|
||||
"NORM_QUANT_READ",
|
||||
"NORM_READ",
|
||||
"PLAIN_RESIDUAL",
|
||||
"ScatterMode",
|
||||
"StageDecl",
|
||||
"StageEntry",
|
||||
"StageInput",
|
||||
"StageOutput",
|
||||
"SumGroup",
|
||||
|
||||
@@ -54,8 +54,16 @@ from sglang.srt.layers.communicator.output import (
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import ResidualOps
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
LayerResidual,
|
||||
StageRead,
|
||||
StageUpdate,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -68,6 +76,8 @@ class StageInput(msgspec.Struct, frozen=True):
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
# How the consumer reads its input from the residual.
|
||||
read: StageRead = NORM_READ
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
@@ -93,6 +103,8 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
# How the producer's output is written into the residual.
|
||||
update: StageUpdate = ADD
|
||||
|
||||
|
||||
class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
@@ -156,7 +168,14 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
construction (``input_owes``); a sum the previous layer leaves for a batch
|
||||
comes with the value."""
|
||||
owes = sides.input_owes is not None
|
||||
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
|
||||
# The previous layer's FFN output, written in as this layer's FFN writes its
|
||||
# own.
|
||||
arrived = StageOutput(
|
||||
sides.input_rows,
|
||||
group=sides.input_owes,
|
||||
always_leaves=owes,
|
||||
update=sides.ffn_output.update,
|
||||
)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
attention_rows = (
|
||||
@@ -187,6 +206,23 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
)
|
||||
|
||||
|
||||
def with_residual(
|
||||
sides: DecoderLayerSides, residual: LayerResidual
|
||||
) -> DecoderLayerSides:
|
||||
"""``sides`` with the reads and updates the layer declares for its
|
||||
attention and its FFN."""
|
||||
replace = msgspec.structs.replace
|
||||
return replace(
|
||||
sides,
|
||||
attention=replace(sides.attention, read=residual.attention_read),
|
||||
attention_output=replace(
|
||||
sides.attention_output, update=residual.attention_update
|
||||
),
|
||||
ffn=replace(sides.ffn, read=residual.ffn_read),
|
||||
ffn_output=replace(sides.ffn_output, update=residual.ffn_update),
|
||||
)
|
||||
|
||||
|
||||
def stage_edges(
|
||||
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
|
||||
) -> Tuple[EdgeDecl, EdgeDecl]:
|
||||
@@ -198,16 +234,18 @@ def stage_edges(
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
update = previous.update if previous is not None else ADD
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
update=update,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows)
|
||||
else StageOutput(rows, update=update)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
@@ -240,7 +278,7 @@ def sequence_parallel_layer_sides(
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers),
|
||||
attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
@@ -281,7 +319,9 @@ def scattered_residual_layer_sides(
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
@@ -315,7 +355,9 @@ def input_scattered_layer_sides(
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
@@ -384,6 +426,7 @@ def decoder_layer_sides(
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
@@ -475,19 +518,33 @@ def _cp_moves() -> CpMoves:
|
||||
)
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
rows the layer hands on."""
|
||||
class StageEntry(msgspec.Struct, frozen=True):
|
||||
"""The boundary into one of a layer's stages, as the layer runs it."""
|
||||
|
||||
# The half into the attention: completes what the input owes, writes the
|
||||
# previous output into the residual and reads the attention input
|
||||
# (_attention_input_step); attention_input then moves it.
|
||||
attention_prepare: Callable
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
# The rows ffn_input hands the FFN.
|
||||
ffn_input_rows: Layout
|
||||
# Completes what the input owes, writes the previous stage's output into
|
||||
# the residual and reads this stage's input:
|
||||
# (hidden_states, residual, forward_batch, norm, context, **call).
|
||||
prepare: Callable
|
||||
# The rows prepare hands the stage.
|
||||
input_rows: Layout
|
||||
# Moves the input onto the stage's rows after prepare, when prepare does
|
||||
# not: (hidden_states, forward_batch, context) -> hidden_states.
|
||||
input_move: Optional[Callable] = None
|
||||
# Hands the stage its input once it is on its rows:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
handoff: Optional[Callable] = None
|
||||
# The fused kernels prepare tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into each of its
|
||||
stages, and the last stage's output on to the rows the layer hands on."""
|
||||
|
||||
# The boundary into the layer's attention and into its FFN; None for a
|
||||
# stage a layer that is one stage does not have.
|
||||
attention: Optional[StageEntry]
|
||||
ffn: Optional[StageEntry]
|
||||
# What the FFN exit reads: the FFN output's group and what it may leave.
|
||||
ffn_output: StageOutput
|
||||
# The postprocess that moves the FFN output on; None when it goes back over
|
||||
@@ -497,11 +554,6 @@ class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
ffn_sum_is_movable: bool
|
||||
# Whether ffn_output_move also completes the sum the FFN leaves.
|
||||
ffn_output_move_completes_sum: bool = False
|
||||
# The fused kernels ffn_input tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Hands the attention its input once attention_input has moved it:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
@@ -520,7 +572,8 @@ def _attention_input_step(
|
||||
layer_input: Optional[Callable],
|
||||
fusions: Tuple[Callable, ...],
|
||||
enters_stack: bool,
|
||||
residual_ops: ResidualOps,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A boundary's half into an attention: complete what the previous layer
|
||||
left (a value that owes a sum, or a producer's handoff one of ``fusions``
|
||||
@@ -549,22 +602,17 @@ def _attention_input_step(
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
if enters:
|
||||
hidden_states, residual = residual_ops.enter(hidden_states), None
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if owed is not None and hidden_states.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return residual_ops.read_attention_input(hidden_states, norm, quant_format)
|
||||
return residual_ops.update_and_read_attention_input(
|
||||
hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
return read.update_and_read(
|
||||
update, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
)
|
||||
|
||||
|
||||
def _another_stage(*args, **kwargs):
|
||||
"""The steps of a stage a single-stage layer does not have."""
|
||||
raise RuntimeError("this layer is one stage and does not have the other")
|
||||
|
||||
|
||||
class InputRead(Enum):
|
||||
"""How a boundary's consumer reads its input from the residual: with the
|
||||
attention input norm (prepare_attn) or with the FFN input norm and its
|
||||
@@ -610,11 +658,10 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the consumer is handed: what it needs, still sharded over
|
||||
the axes it gathers itself."""
|
||||
the axes it gathers itself as the rows its input is read on are."""
|
||||
need = self.edge.need
|
||||
return Layout(
|
||||
need.layout.sharded
|
||||
| (self.edge.produced.layout.sharded & need.gathers_itself)
|
||||
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
|
||||
)
|
||||
|
||||
|
||||
@@ -625,7 +672,6 @@ def make_boundary(
|
||||
fusions: Tuple = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""The steps a layer runs at ``edge``, around the residual operations;
|
||||
@@ -646,7 +692,7 @@ def make_boundary(
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
@@ -663,7 +709,8 @@ def make_boundary(
|
||||
fusions=fusions,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
read=edge.need.read,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(edge, prepare=input_step, fused=fused)
|
||||
layer_input = None
|
||||
@@ -683,9 +730,15 @@ def make_boundary(
|
||||
prepare=partial(
|
||||
_attention_input_step,
|
||||
layer_input=layer_input,
|
||||
fusions=fusions,
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
fusions=(
|
||||
fusions
|
||||
if edge.produced.update.adds_plainly and edge.need.read.norms_plainly
|
||||
else ()
|
||||
),
|
||||
enters_stack=enters_stack,
|
||||
residual_ops=residual_ops,
|
||||
read=edge.need.read,
|
||||
update=edge.produced.update,
|
||||
),
|
||||
input_move=_select_attention_input_move(edge.residual_to, edge.need),
|
||||
)
|
||||
@@ -697,7 +750,6 @@ def _select_boundary_steps(
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input,
|
||||
attention_fusions: Tuple[Callable, ...] = (),
|
||||
enters_stack: bool = False,
|
||||
@@ -708,35 +760,36 @@ def _select_boundary_steps(
|
||||
input tries ``attention_fusions``, and the layer stack's first layer
|
||||
(``enters_stack``) starts its residual there."""
|
||||
edges = decoder_layer_edges(sides)
|
||||
out_of_ffn = make_boundary(
|
||||
edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops
|
||||
)
|
||||
out_of_ffn = make_boundary(edges.out_of_ffn, reads=None, cp_moves=cp_moves)
|
||||
into_ffn = make_boundary(
|
||||
edges.into_ffn,
|
||||
reads=InputRead.FFN,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
into_attention = make_boundary(
|
||||
edges.into_attention,
|
||||
reads=InputRead.ATTENTION,
|
||||
fusions=attention_fusions,
|
||||
residual_ops=residual_ops,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_prepare=into_attention.prepare,
|
||||
attention_input=into_attention.input_move,
|
||||
ffn_input=into_ffn.prepare,
|
||||
ffn_input_rows=into_ffn.input_rows,
|
||||
attention=StageEntry(
|
||||
prepare=into_attention.prepare,
|
||||
input_rows=into_attention.input_rows,
|
||||
input_move=into_attention.input_move,
|
||||
handoff=attention_handoff,
|
||||
),
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
input_rows=into_ffn.input_rows,
|
||||
fused=into_ffn.fused,
|
||||
),
|
||||
ffn_output=edges.out_of_ffn.produced,
|
||||
ffn_output_move=out_of_ffn.output_move,
|
||||
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
|
||||
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
|
||||
fused=into_ffn.fused,
|
||||
attention_handoff=attention_handoff,
|
||||
)
|
||||
|
||||
|
||||
@@ -765,7 +818,8 @@ def _select_ffn_input(
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
@@ -791,7 +845,8 @@ def _select_ffn_input(
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
read=read,
|
||||
update=update,
|
||||
)
|
||||
return partial(_mlp_input_completing_owed, step=step), fused
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
@@ -819,7 +874,8 @@ def _select_ffn_input(
|
||||
partial(
|
||||
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
|
||||
scatters_residual=residual != residual_to,
|
||||
residual_ops=residual_ops,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -828,7 +884,10 @@ def _select_ffn_input(
|
||||
# attention's rows (MHC on an input-scattered batch).
|
||||
if gathered or not owes_attention_tp or residual != residual_to:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
|
||||
return (
|
||||
partial(_mlp_input_on_residual_shard, read=read, update=update),
|
||||
(),
|
||||
)
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
@@ -841,7 +900,8 @@ def _select_ffn_input(
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
residual_ops=residual_ops,
|
||||
read=read,
|
||||
update=update,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused
|
||||
if (
|
||||
@@ -860,20 +920,26 @@ def _select_ffn_input(
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
|
||||
return partial(_mlp_input_norm, read=read, update=update), ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if not residual_ops.adds_plainly:
|
||||
if not update.adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return partial(_mlp_input_residual_into_sum, read=read), ()
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
fused = (
|
||||
tuple(f for f in fusions if f.completes is produced.group)
|
||||
if update.adds_plainly and read.norms_plainly
|
||||
else ()
|
||||
)
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
residual_ops=residual_ops,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
fused,
|
||||
)
|
||||
@@ -886,13 +952,15 @@ def _select_ffn_input(
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and residual_ops.adds_plainly
|
||||
and update.adds_plainly
|
||||
and read.norms_plainly
|
||||
):
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_dp_partial,
|
||||
gathers_residual=gathers_residual,
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -902,7 +970,8 @@ def _select_ffn_input(
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
places_cp_shards=places_cp_shards,
|
||||
residual_ops=residual_ops,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -914,7 +983,7 @@ def _select_ffn_output_move(
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
update: StageUpdate = ADD,
|
||||
) -> Tuple[bool, Optional[Callable], bool]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
@@ -928,7 +997,7 @@ def _select_ffn_output_move(
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return False, partial(pair._gather, residual_ops=residual_ops), False
|
||||
return False, partial(pair._gather, update=update), False
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
|
||||
@@ -942,7 +1011,7 @@ def _select_ffn_output_move(
|
||||
pair._onto_residual_shard,
|
||||
sums=sums,
|
||||
gathers_back=to != residual,
|
||||
residual_ops=residual_ops,
|
||||
update=update,
|
||||
),
|
||||
sums,
|
||||
)
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import partial
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
FusedMlpInput,
|
||||
InputRead,
|
||||
LayerStage,
|
||||
_another_stage,
|
||||
StageEntry,
|
||||
_cp_moves,
|
||||
_select_boundary_steps,
|
||||
decoder_layer_sides,
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
make_boundary,
|
||||
scattered_residual_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
with_residual,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
@@ -73,9 +74,9 @@ from sglang.srt.layers.communicator.output import (
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import ResidualOps
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD_AND_NORM,
|
||||
PLAIN_RESIDUAL,
|
||||
aiter_all_reduce_fusion_enabled_for,
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
@@ -244,13 +245,68 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce()
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
|
||||
self._layer = layer
|
||||
self._name = name
|
||||
self._norm_name = norm_name
|
||||
|
||||
@property
|
||||
def norm(self) -> torch.nn.Module:
|
||||
"""The norm the stage reads its input with, one of the layer's."""
|
||||
return getattr(self._layer, self._norm_name)
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
entry = getattr(steps, self._name)
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self._name} stage"
|
||||
)
|
||||
return entry
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
context = self._layer._context
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, context, **call
|
||||
)
|
||||
if entry.input_move is not None:
|
||||
hidden_states = entry.input_move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=context,
|
||||
)
|
||||
if entry.handoff is not None:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self._layer.qkv_latent_func
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
# layout and try no fused kernel at the FFN exit.
|
||||
_publish_lora_layout: bool = False
|
||||
_ffn_exit_fusions: Tuple[Callable, ...] = ()
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual_ops: ResidualOps = ADD_AND_NORM
|
||||
_residual: LayerResidual = PLAIN_RESIDUAL
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -265,8 +321,9 @@ class LayerCommunicator:
|
||||
fused_ar_quant_keep_bf16: bool = False,
|
||||
# False for a layer whose FFN always completes its own reduction.
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
# How the layer writes its residual and reads its stages' inputs.
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
# How the layer's attention and FFN read their inputs from the residual
|
||||
# and write their outputs into it.
|
||||
residual: LayerResidual = PLAIN_RESIDUAL,
|
||||
# A layer that is one stage of a sequence of stages, instead of an
|
||||
# attention followed by an FFN.
|
||||
stage: Optional["LayerStage"] = None,
|
||||
@@ -281,7 +338,7 @@ class LayerCommunicator:
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
self._residual_ops = residual_ops
|
||||
self._residual = residual
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -299,6 +356,11 @@ class LayerCommunicator:
|
||||
get_lora().enable_lora
|
||||
)
|
||||
if stage is not None:
|
||||
if residual != PLAIN_RESIDUAL:
|
||||
raise ValueError(
|
||||
"a layer that is one stage takes its reads and updates from "
|
||||
"the stage's declarations"
|
||||
)
|
||||
self._init_stage(stage)
|
||||
return
|
||||
# Its two boundaries, for a layer that is one stage.
|
||||
@@ -336,8 +398,10 @@ class LayerCommunicator:
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
|
||||
residual_ops=residual_ops,
|
||||
with_residual(
|
||||
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
|
||||
residual,
|
||||
),
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
)
|
||||
@@ -354,9 +418,9 @@ class LayerCommunicator:
|
||||
)
|
||||
self._declared = None
|
||||
self._cp_steps = self._input_scattered_steps = self._sp_steps = None
|
||||
self.stage_edges = stage.edges
|
||||
into_edge, out_edge = stage.edges
|
||||
reads_ffn = stage.reads is InputRead.FFN
|
||||
self.stage_edges = stage.edges
|
||||
into = make_boundary(
|
||||
into_edge,
|
||||
reads=stage.reads,
|
||||
@@ -366,22 +430,35 @@ class LayerCommunicator:
|
||||
else self._attn_input_fusions
|
||||
),
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
residual_ops=self._residual_ops,
|
||||
enters_stack=stage.enters_stack,
|
||||
)
|
||||
out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops)
|
||||
out = make_boundary(out_edge, reads=None)
|
||||
entry = StageEntry(
|
||||
prepare=into.prepare,
|
||||
input_rows=into.input_rows,
|
||||
input_move=None if reads_ffn else into.input_move,
|
||||
handoff=None if reads_ffn else _hand_qkv_hook_its_input,
|
||||
fused=into.fused,
|
||||
)
|
||||
self._steps = BoundarySteps(
|
||||
attention_prepare=_another_stage if reads_ffn else into.prepare,
|
||||
attention_input=_another_stage if reads_ffn else into.input_move,
|
||||
ffn_input=into.prepare if reads_ffn else _another_stage,
|
||||
ffn_input_rows=into.input_rows,
|
||||
attention=None if reads_ffn else entry,
|
||||
ffn=entry if reads_ffn else None,
|
||||
ffn_output=out_edge.produced,
|
||||
ffn_output_move=out.output_move,
|
||||
ffn_output_move_completes_sum=out.output_move_completes_sum,
|
||||
ffn_sum_is_movable=out_edge.produced.group is not None,
|
||||
fused=into.fused,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def attn(self) -> StageCommunicator:
|
||||
"""The layer's attention stage."""
|
||||
return StageCommunicator(self, "attention", "input_layernorm")
|
||||
|
||||
@cached_property
|
||||
def ffn(self) -> StageCommunicator:
|
||||
"""The layer's FFN stage."""
|
||||
return StageCommunicator(self, "ffn", "post_attention_layernorm")
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the layer's input and residual arrive on in a batch that
|
||||
@@ -472,7 +549,7 @@ class LayerCommunicator:
|
||||
# No batch shards its tokens and there is no attention DP: as
|
||||
# without CP.
|
||||
may_leave = may_leave_to_reduce_scatter = True
|
||||
return decoder_layer_sides(
|
||||
sides = decoder_layer_sides(
|
||||
axis_sizes=token_axis_sizes(cp_active=cp_active),
|
||||
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
|
||||
ffn_shards_over_cp=ffn_on_cp_shards,
|
||||
@@ -499,6 +576,7 @@ class LayerCommunicator:
|
||||
)
|
||||
and may_leave,
|
||||
)
|
||||
return with_residual(sides, self._residual)
|
||||
|
||||
@staticmethod
|
||||
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
@@ -532,7 +610,7 @@ class LayerCommunicator:
|
||||
for the layer's ordinary declarations ``sides``. A plain residual comes
|
||||
back to the full rows inside the attention output's sum; one whose
|
||||
write-back is not a plain add stays on each rank's slice."""
|
||||
if self._residual_ops.adds_plainly:
|
||||
if self._residual.attention_update.adds_plainly:
|
||||
scattered = input_scattered_layer_sides(
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
@@ -550,8 +628,7 @@ class LayerCommunicator:
|
||||
# DSA and hook-less attention take the slice gathered.
|
||||
handoff = _hand_scattered_input_to_attention
|
||||
return _select_boundary_steps(
|
||||
scattered,
|
||||
residual_ops=self._residual_ops,
|
||||
with_residual(scattered, self._residual),
|
||||
attention_handoff=handoff,
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
@@ -563,7 +640,6 @@ class LayerCommunicator:
|
||||
"""The steps this layer runs for a set of declarations."""
|
||||
return _select_boundary_steps(
|
||||
sides,
|
||||
residual_ops=self._residual_ops,
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
**kwargs,
|
||||
@@ -586,9 +662,7 @@ class LayerCommunicator:
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
gathered_last_layer_output = self._batch_steps(
|
||||
forward_batch
|
||||
).attention_input(
|
||||
gathered_last_layer_output = self.attn.entry(forward_batch).input_move(
|
||||
hidden_states=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
@@ -616,10 +690,7 @@ class LayerCommunicator:
|
||||
the batch is not input-scattered.
|
||||
"""
|
||||
return (
|
||||
any(
|
||||
f.may_return_new_residual
|
||||
for f in self._batch_steps(forward_batch).fused
|
||||
)
|
||||
any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused)
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
@@ -637,7 +708,7 @@ class LayerCommunicator:
|
||||
"lora_batch_layout",
|
||||
(
|
||||
LoRABatchLayout.DP_LOCAL
|
||||
if TokenAxis.ATTN_DP in steps.ffn_input_rows.sharded
|
||||
if TokenAxis.ATTN_DP in steps.ffn.input_rows.sharded
|
||||
else LoRABatchLayout.TP_GLOBAL
|
||||
),
|
||||
)
|
||||
@@ -660,20 +731,13 @@ class LayerCommunicator:
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
hidden_states, residual = self._batch_steps(forward_batch).attention_prepare(
|
||||
return self.attn.prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.input_layernorm,
|
||||
self._context,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_steps is not None and get_forward().sp_active
|
||||
@@ -693,27 +757,14 @@ class LayerCommunicator:
|
||||
return self._cp_steps
|
||||
return self._steps
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
steps = self._batch_steps(forward_batch)
|
||||
hidden_states = steps.attention_input(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
hidden_states = steps.attention_handoff(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete what the previous layer left together
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly."""
|
||||
plainly: what arrives is written in as this layer's FFN writes its own."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
self._residual.ffn_update.adds_plainly
|
||||
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
@@ -772,7 +823,7 @@ class LayerCommunicator:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
self._residual.attention_update.adds_plainly
|
||||
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
@@ -813,13 +864,7 @@ class LayerCommunicator:
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
return steps.ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.post_attention_layernorm,
|
||||
self._context,
|
||||
)
|
||||
return self.ffn.prepare(hidden_states, residual, forward_batch, steps)
|
||||
|
||||
def maybe_prefetch_next_full_attention_kv(
|
||||
self,
|
||||
@@ -847,10 +892,8 @@ class LayerCommunicator:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
the next layer's input also writes the output into the residual."""
|
||||
return (
|
||||
self._batch_steps(forward_batch).returns_over_dp
|
||||
and not self._residual_ops.updates_residual_after_ffn
|
||||
)
|
||||
steps = self._batch_steps(forward_batch)
|
||||
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -977,8 +1020,9 @@ class LayerCommunicator:
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
)
|
||||
if residual is not None and self._residual_ops.updates_residual_after_ffn:
|
||||
hidden_states = self._residual_ops.update_residual(hidden_states, residual)
|
||||
update = steps.ffn_output.update
|
||||
if residual is not None and update.at_producer:
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -1326,7 +1370,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
allow_reduce_scatter,
|
||||
qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
residual_ops=self.mhc,
|
||||
residual=self.mhc.layer_residual(),
|
||||
)
|
||||
# MHC has not been run with input-scattered attention under attention CP.
|
||||
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
|
||||
|
||||
@@ -41,8 +41,8 @@ from sglang.srt.layers.communicator.layout import (
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output
|
||||
from sglang.srt.layers.communicator.residual import ResidualOps
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
@@ -249,21 +249,22 @@ def _mlp_input_without_dp(
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
if gathers_residual:
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
@@ -276,14 +277,15 @@ def _mlp_input_dp_replicate(
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
@@ -291,12 +293,12 @@ def _mlp_input_dp_replicate(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
@@ -314,10 +316,11 @@ def _mlp_input_dp_partial(
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then
|
||||
normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP
|
||||
group's tokens beside the others' in the group's slot."""
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then read
|
||||
the input from the sum. With ``places_cp_shards`` each CP rank puts its
|
||||
shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
if context.attn_tp_rank == 0:
|
||||
@@ -327,8 +330,7 @@ def _mlp_input_dp_partial(
|
||||
hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = layernorm(hidden_states)
|
||||
hidden_states, _ = read.read(hidden_states, layernorm)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -402,9 +404,10 @@ def _mlp_input_norm(
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
@@ -415,14 +418,15 @@ def _mlp_input_scatter(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states, context
|
||||
)
|
||||
if scatters_residual:
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
residual = update.residual_to_attn_tp_shard(residual, context)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_slice(
|
||||
@@ -433,12 +437,13 @@ def _mlp_input_slice(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
residual = update.residual_to_attn_tp_shard(residual, context)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
@@ -448,7 +453,8 @@ def _mlp_input_on_residual_shard(
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
@@ -458,7 +464,7 @@ def _mlp_input_on_residual_shard(
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
|
||||
shard, residual = read.update_and_read(update, shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -469,9 +475,11 @@ def _mlp_input_residual_into_sum(
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context
|
||||
hidden_states, residual, layernorm, context, read
|
||||
)
|
||||
|
||||
|
||||
@@ -480,6 +488,7 @@ def _tp_all_reduce_with_scattered_residual(
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
@@ -487,8 +496,7 @@ def _tp_all_reduce_with_scattered_residual(
|
||||
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
scattered_states += residual
|
||||
residual = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = layernorm(residual)
|
||||
return hidden_states, residual
|
||||
return read.read(residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_gather_attention_cp(
|
||||
@@ -669,10 +677,10 @@ class CommunicateSummableTensorPairFn:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
update: StageUpdate = ADD,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = residual_ops.update_residual(hidden_states, residual)
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
@@ -684,7 +692,7 @@ class CommunicateSummableTensorPairFn:
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
residual_ops: ResidualOps,
|
||||
update: StageUpdate,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
@@ -697,7 +705,7 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = residual_ops.update_residual(hidden_states, residual)
|
||||
local_states = update.update(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
|
||||
@@ -11,50 +11,77 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""How a layer writes its stages' outputs into the residual and reads the next
|
||||
stage's input back."""
|
||||
"""How a stage writes its output into the residual (its update) and how a
|
||||
stage reads its input back from it (its read). The producer of a boundary
|
||||
declares the update, the consumer the read; the boundary steps complete sums
|
||||
and move tokens around the two."""
|
||||
|
||||
from typing import Protocol, Tuple
|
||||
from typing import NamedTuple, Optional, Protocol, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class ResidualOps(Protocol):
|
||||
"""How a layer writes each stage's output into its residual and reads the
|
||||
next stage's input from it. The boundary steps complete sums and move
|
||||
tokens around these operations. AddAndNorm adds and normalizes; MHC's
|
||||
hyper-connection streams implement the same operations (residual.mhc)."""
|
||||
class StageUpdate(Protocol):
|
||||
"""How a stage's output is written into the residual."""
|
||||
|
||||
# The write-back is a plain add, which one rank may run before the sum it
|
||||
# adds into completes: the DP partial order, and a residual that joins the
|
||||
# attention output's sum.
|
||||
# A plain add, which one rank may run before the sum it adds into
|
||||
# completes (the DP partial order, a residual that joins the attention
|
||||
# output's sum), and which a fused add + norm kernel may run.
|
||||
adds_plainly: bool
|
||||
# The layer writes its FFN output into the residual itself instead of
|
||||
# leaving that to the next layer's input.
|
||||
updates_residual_after_ffn: bool
|
||||
# The stage writes its output into the residual itself at its end, instead
|
||||
# of leaving that to the next stage's read.
|
||||
at_producer: bool
|
||||
|
||||
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
|
||||
"""The attention input and the residual, from a residual that already
|
||||
holds the previous layer's output."""
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
|
||||
) -> Tuple:
|
||||
"""Write the previous layer's output into the residual, then read the
|
||||
attention input from it."""
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
|
||||
"""Write the attention output into the residual, then read the FFN
|
||||
input from it."""
|
||||
|
||||
def update_residual(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the FFN output written into it."""
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the output written into it."""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual."""
|
||||
"""This attention-TP rank's slice of the residual, with whatever the
|
||||
update reads along with it."""
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
|
||||
"""The residual gathered from every attention-TP rank's slice."""
|
||||
|
||||
|
||||
class StageRead(Protocol):
|
||||
"""How a stage reads its input from the residual."""
|
||||
|
||||
# The input is the residual's norm, in the quantization the call asks for,
|
||||
# and the residual is left as it is: what a fused add + norm kernel computes,
|
||||
# and what may run on the rows a sum completes onto.
|
||||
norms_plainly: bool
|
||||
|
||||
def enter(self, hidden_states) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
def read(
|
||||
self,
|
||||
residual,
|
||||
norm,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple:
|
||||
"""The input and the residual, from a residual that already holds the
|
||||
previous stage's output."""
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
update: StageUpdate,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple:
|
||||
"""Write the previous stage's output into the residual with its
|
||||
producer's ``update``, then read the input."""
|
||||
|
||||
|
||||
class LayerResidual(NamedTuple):
|
||||
"""The reads and updates a decoder layer declares for its attention and
|
||||
its FFN."""
|
||||
|
||||
attention_read: StageRead
|
||||
attention_update: StageUpdate
|
||||
ffn_read: StageRead
|
||||
ffn_update: StageUpdate
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.adapters.attention import (
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
is_flashinfer_allreduce_unavailable,
|
||||
@@ -261,43 +262,13 @@ def _attn_input_update_and_read_residual(quant_format: str):
|
||||
return _update_and_read_residual_plain
|
||||
|
||||
|
||||
class AddAndNorm:
|
||||
"""A plain residual: a stage's output is added into it, and the next
|
||||
stage's input is its norm. An empty batch skips the norms."""
|
||||
class Add:
|
||||
"""A plain residual: the output is added into it."""
|
||||
|
||||
adds_plainly = True
|
||||
updates_residual_after_ffn = False
|
||||
at_producer = False
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, residual, None, None
|
||||
)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
if residual is None:
|
||||
# The layer stack starts at this FFN: its input is the residual.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return norm(hidden_states), hidden_states
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
@@ -308,4 +279,91 @@ class AddAndNorm:
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
ADD_AND_NORM = AddAndNorm()
|
||||
class NormQuantRead:
|
||||
"""The input is the residual's norm, fused with the quantization the
|
||||
consumer asks for (``quant_format``) and a post-residual addition; a plain
|
||||
add of the previous output runs in the same kernel. An empty batch skips
|
||||
the norm."""
|
||||
|
||||
norms_plainly = True
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, residual, None, None
|
||||
)
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format="",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if not update.adds_plainly:
|
||||
return self.read(update.update(hidden_states, residual), norm, quant_format)
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
)
|
||||
|
||||
|
||||
class NormRead:
|
||||
"""The input is the residual's norm; a plain add of the previous output
|
||||
runs in the same kernel. An empty batch skips the norm."""
|
||||
|
||||
norms_plainly = True
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if quant_format or post_residual_addition is not None:
|
||||
raise NotImplementedError(
|
||||
f"a norm read with {quant_format=} or a post-residual addition"
|
||||
)
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return norm(residual), residual
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format="",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if residual is None:
|
||||
# The layer stack starts at this stage: its input is the residual.
|
||||
return self.read(hidden_states, norm, quant_format, post_residual_addition)
|
||||
if not update.adds_plainly:
|
||||
return self.read(update.update(hidden_states, residual), norm, quant_format)
|
||||
if quant_format or post_residual_addition is not None:
|
||||
raise NotImplementedError(
|
||||
f"a norm read with {quant_format=} or a post-residual addition"
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
|
||||
ADD = Add()
|
||||
NORM_QUANT_READ = NormQuantRead()
|
||||
NORM_READ = NormRead()
|
||||
# A plain residual: the attention reads with its input norm and the quantization
|
||||
# it wants, the FFN with its norm, and each stage's output is added.
|
||||
PLAIN_RESIDUAL = LayerResidual(
|
||||
attention_read=NORM_QUANT_READ,
|
||||
attention_update=ADD,
|
||||
ffn_read=NORM_READ,
|
||||
ffn_update=ADD,
|
||||
)
|
||||
|
||||
@@ -19,17 +19,18 @@ from typing import Callable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
|
||||
|
||||
@dataclass
|
||||
class MHCState:
|
||||
"""A layer's residual as hyper-connection streams, with the residual
|
||||
operations the boundary steps run (communicator.ResidualOps): the residual
|
||||
is hc_mult streams, a stage's output is written in with hc_post and the
|
||||
next stage's input read with hc_pre and the norm. A read produces the
|
||||
h_res / h_post the next write-back consumes; they move with the tokens.
|
||||
Parameters belong to the owning layer; this state only holds scratch
|
||||
shared across communication stages."""
|
||||
"""A layer's residual as hyper-connection streams: the residual is hc_mult
|
||||
streams, a stage's output is written in with hc_post and the next stage's
|
||||
input read with hc_pre and the norm. A read produces the h_res / h_post the
|
||||
next write-back consumes; they move with the tokens. ``layer_residual()``
|
||||
gives the reads and updates the layer's two stages declare. Parameters
|
||||
belong to the owning layer; this state only holds scratch shared across
|
||||
communication stages."""
|
||||
|
||||
hc_mult: int
|
||||
hc_attn_pre: Callable
|
||||
@@ -42,13 +43,6 @@ class MHCState:
|
||||
h_res: Optional[torch.Tensor] = None
|
||||
h_post: Optional[torch.Tensor] = None
|
||||
|
||||
# hc_post is not a plain add, so it runs only once the sum it writes in
|
||||
# is complete.
|
||||
adds_plainly = False
|
||||
# The FFN output's write-back uses the coefficients this layer's FFN input
|
||||
# read produced, so the layer runs it itself.
|
||||
updates_residual_after_ffn = True
|
||||
|
||||
@staticmethod
|
||||
def _resolve_out_norm(out_norm):
|
||||
if out_norm is None:
|
||||
@@ -102,30 +96,6 @@ class MHCState:
|
||||
self.h_res = None
|
||||
self.h_post = None
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hc_expand(hidden_states, self.hc_mult)
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if quant_format:
|
||||
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
|
||||
return self.attn_split(residual, out_norm=norm)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
# An MHC layer's input arrives with the previous FFN output written in.
|
||||
raise NotImplementedError("an MHC layer takes its input written back")
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
return self.attn_to_mlp(hidden_states, residual, out_norm=norm)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states = self.mlp_combine(hidden_states, residual)
|
||||
self.reset_aux()
|
||||
if self.is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, self.hc_mult)
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
rank, size = context.attn_tp_rank, context.attn_tp_size
|
||||
self.h_res = self.h_res.tensor_split(size)[rank]
|
||||
@@ -136,3 +106,98 @@ class MHCState:
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
|
||||
def layer_residual(self) -> LayerResidual:
|
||||
return LayerResidual(
|
||||
attention_read=_AttentionRead(self),
|
||||
attention_update=_AttentionUpdate(self),
|
||||
ffn_read=_FfnRead(self),
|
||||
ffn_update=_FfnUpdate(self),
|
||||
)
|
||||
|
||||
|
||||
class _AttentionRead:
|
||||
"""hc_pre and the input norm, from streams that already hold the previous
|
||||
layer's output: an MHC layer takes its input written back."""
|
||||
|
||||
norms_plainly = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hc_expand(hidden_states, self.state.hc_mult)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
if quant_format:
|
||||
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
|
||||
return self.state.attn_split(residual, out_norm=norm)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
raise NotImplementedError("an MHC layer takes its input written back")
|
||||
|
||||
|
||||
class _AttentionUpdate:
|
||||
"""hc_post with the coefficients the attention's read produced. It is not a
|
||||
plain add, so it runs only once the sum it writes in is complete."""
|
||||
|
||||
adds_plainly = False
|
||||
at_producer = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return self.state.mlp_combine(hidden_states, residual)
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return self.state.residual_to_attn_tp_shard(residual, context)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return self.state.residual_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
class _FfnRead:
|
||||
"""The attention output's hc_post and the FFN input's hc_pre and norm, fused
|
||||
in hc_ffn_post_pre when it takes the batch."""
|
||||
|
||||
norms_plainly = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hc_expand(hidden_states, self.state.hc_mult)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
raise NotImplementedError("an MHC FFN input read without its attention")
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
if not isinstance(update, _AttentionUpdate) or update.state is not self.state:
|
||||
raise NotImplementedError(f"an MHC FFN input after {update=}")
|
||||
return self.state.attn_to_mlp(hidden_states, residual, out_norm=norm)
|
||||
|
||||
|
||||
class _FfnUpdate:
|
||||
"""hc_post with the coefficients the FFN input's read produced, which this
|
||||
layer runs itself; the last layer also contracts the streams into the
|
||||
hidden states the layer stack hands on."""
|
||||
|
||||
adds_plainly = False
|
||||
at_producer = True
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states = self.state.mlp_combine(hidden_states, residual)
|
||||
self.state.reset_aux()
|
||||
if self.state.is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, self.state.hc_mult)
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return self.state.residual_to_attn_tp_shard(residual, context)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return self.state.residual_from_attn_tp_shards(residual)
|
||||
|
||||
@@ -361,7 +361,7 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
and self._context.tp_size > 1
|
||||
# The FFN runs on the full rows, not each rank's own slice.
|
||||
and TokenAxis.ATTN_TP_SCATTER
|
||||
not in self._batch_steps(forward_batch).ffn_input_rows.sharded
|
||||
not in self._batch_steps(forward_batch).ffn.input_rows.sharded
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Optional
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
InputRead,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
@@ -37,11 +38,12 @@ def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]:
|
||||
|
||||
|
||||
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
"""What the stage at ``layer_idx`` declares. A mixer computes on the
|
||||
attention's rows and leaves its attention-TP sum to an FFN stage after it,
|
||||
or, when a fused kernel takes it, to a mixer after it. An FFN computes on
|
||||
the TP group's rows, a MoE dispatched by an a2a backend on this rank's own,
|
||||
and may leave its sum to a mixer after it."""
|
||||
"""What the stage at ``layer_idx`` declares. A mixer reads its input as
|
||||
an attention does, computes on the attention's rows and leaves its
|
||||
attention-TP sum to an FFN stage after it, or, when a fused kernel takes
|
||||
it, to a mixer after it. An FFN computes on the TP group's rows, a MoE
|
||||
dispatched by an a2a backend on this rank's own, and may leave its sum to a
|
||||
mixer after it."""
|
||||
axis_sizes = token_axis_sizes()
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
@@ -50,7 +52,7 @@ def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
if is_attn_layer(pattern[layer_idx]):
|
||||
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return StageDecl(
|
||||
StageInput(attention),
|
||||
StageInput(attention, read=NORM_QUANT_READ),
|
||||
StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
|
||||
@@ -10,7 +10,7 @@ from unittest import mock
|
||||
|
||||
from sglang.srt.layers import communicator
|
||||
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
|
||||
from sglang.srt.layers.communicator import residual as residual_ops
|
||||
from sglang.srt.layers.communicator import residual as residual_contract
|
||||
from sglang.srt.layers.communicator.adapters import attention
|
||||
from sglang.srt.layers.communicator.residual import add_norm, mhc
|
||||
|
||||
@@ -19,7 +19,7 @@ COMMUNICATOR_MODULES = (
|
||||
layout,
|
||||
output,
|
||||
attention,
|
||||
residual_ops,
|
||||
residual_contract,
|
||||
add_norm,
|
||||
mhc,
|
||||
ops,
|
||||
|
||||
Reference in New Issue
Block a user