[Refactor] Build the boundary into any stage with one construction (#41441)
This commit is contained in:
@@ -22,16 +22,17 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
DecoderLayerSides,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
InputRead,
|
||||
LayerStage,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
decoder_layer_edges,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
sequence_parallel_layer_sides,
|
||||
stage_edges,
|
||||
tbo_split_moves,
|
||||
@@ -95,7 +96,6 @@ __all__ = [
|
||||
"FfnExitFusion",
|
||||
"FusedMlpInput",
|
||||
"HandoffOutput",
|
||||
"InputRead",
|
||||
"LayerCommunicator",
|
||||
"LayerResidual",
|
||||
"LayerScatterModes",
|
||||
@@ -111,6 +111,7 @@ __all__ = [
|
||||
"StageDecl",
|
||||
"StageEntry",
|
||||
"StageInput",
|
||||
"StageKind",
|
||||
"StageOutput",
|
||||
"SumGroup",
|
||||
"TokenAxis",
|
||||
@@ -122,6 +123,7 @@ __all__ = [
|
||||
"input_scattered_layer_sides",
|
||||
"layer_input_buffer",
|
||||
"make_boundary",
|
||||
"make_output_boundary",
|
||||
"moe_cp_gathers_sparse_moe_input",
|
||||
"move_rows",
|
||||
"reduce_output",
|
||||
|
||||
@@ -16,14 +16,13 @@ declarations."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from functools import partial
|
||||
from typing import Callable, FrozenSet, Mapping, Optional, Tuple, Union
|
||||
from typing import Callable, FrozenSet, Mapping, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -35,25 +34,20 @@ from sglang.srt.layers.communicator.layout import (
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
_consumer_step,
|
||||
_hand_qkv_hook_its_input,
|
||||
_mlp_input_completing_owed,
|
||||
_mlp_input_dp_partial,
|
||||
_mlp_input_dp_replicate,
|
||||
_mlp_input_gather_attention_cp,
|
||||
_mlp_input_gather_moe_cp,
|
||||
_mlp_input_norm,
|
||||
_mlp_input_on_residual_shard,
|
||||
_mlp_input_residual_into_sum,
|
||||
_mlp_input_scatter,
|
||||
_mlp_input_slice,
|
||||
_mlp_input_without_dp,
|
||||
_read_input,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
LayerResidual,
|
||||
StageRead,
|
||||
@@ -64,7 +58,6 @@ from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@@ -560,63 +553,10 @@ class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
return self.ffn_output_move is None
|
||||
|
||||
|
||||
def _attention_input_step(
|
||||
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
context: "CommunicateContext",
|
||||
*,
|
||||
quant_format: str,
|
||||
post_residual_addition: Optional[torch.Tensor],
|
||||
layer_input: Optional[Callable],
|
||||
fusions: Tuple[Callable, ...],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A boundary's half into an attention: complete what the previous layer
|
||||
left (a value that owes a sum, or a producer's handoff one of ``fusions``
|
||||
consumes with the add and norm), what the input owes by construction
|
||||
(``layer_input``), then write the previous output into the residual and read
|
||||
the attention input with ``norm``. The layer stack's first layer
|
||||
(``enters_stack``) starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
|
||||
if owed is not None and residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if isinstance(owed, UnreducedOutput) and owed.reduce_and_redistribute is not None:
|
||||
# No fused kernel runs under attention DP: the reduce-scatter back to
|
||||
# this rank's tokens comes first.
|
||||
hidden_states, owed = reduce_output(owed), None
|
||||
if owed is not None:
|
||||
for fused in fusions:
|
||||
result = fused(owed, residual, forward_batch, post_residual_addition)
|
||||
if result is not None:
|
||||
return result
|
||||
if isinstance(owed, HandoffOutput):
|
||||
# No fused kernel took the handoff: its producer completes it.
|
||||
hidden_states, owed = reduce_output(owed), None
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
if enters:
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if owed is not None and hidden_states.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
return read.update_and_read(
|
||||
update, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
)
|
||||
|
||||
|
||||
class InputRead(Enum):
|
||||
"""How a boundary's consumer reads its input from the residual: with the
|
||||
attention input norm (prepare_attn) or with the FFN input norm and its
|
||||
fused kernels (prepare_mlp)."""
|
||||
class StageKind(Enum):
|
||||
"""Which of a decoder layer's two stages a layer that is one stage takes
|
||||
the place of: its norm, its read and update, its fused kernels and its
|
||||
entry in the layer's steps."""
|
||||
|
||||
ATTENTION = auto()
|
||||
FFN = auto()
|
||||
@@ -624,12 +564,12 @@ class InputRead(Enum):
|
||||
|
||||
class LayerStage(msgspec.Struct, frozen=True):
|
||||
"""A layer that is one stage of a sequence of stages, each an
|
||||
attention-like mixer or an FFN: how it reads its input, its two boundaries
|
||||
attention-like mixer or an FFN: which of the two it is, its two boundaries
|
||||
(into it, and out of it onto the rows every layer hands on, as
|
||||
``stage_edges`` gives them), and whether the layer stack
|
||||
starts at it."""
|
||||
|
||||
reads: InputRead
|
||||
kind: StageKind
|
||||
edges: Tuple[EdgeDecl, EdgeDecl]
|
||||
enters_stack: bool = False
|
||||
|
||||
@@ -642,11 +582,11 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
|
||||
edge: EdgeDecl
|
||||
# The consumer's half: completing what the input owes, the add and the
|
||||
# norm; into the FFN also the move onto the rows it needs.
|
||||
# norm, and the moves onto the rows it needs that come before the read.
|
||||
prepare: Optional[Callable] = None
|
||||
# Into an attention, the move onto its rows after prepare.
|
||||
# The move onto the consumer's rows after prepare; None when there is none.
|
||||
input_move: Optional[Callable] = None
|
||||
# The fused kernels an FFN's prepare tries first.
|
||||
# The fused kernels prepare tries first on a sum the input always owes.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# The producer's half: the postprocess that moves the output onto the rows
|
||||
# the layer hands on; None when it goes back over attention DP, whose step
|
||||
@@ -668,79 +608,70 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
def make_boundary(
|
||||
edge: EdgeDecl,
|
||||
*,
|
||||
reads: Optional[InputRead],
|
||||
fusions: Tuple = (),
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""The steps a layer runs at ``edge``, around the residual operations;
|
||||
``cp_moves`` for an edge that gathers over or returns across attention CP.
|
||||
``reads`` is how the consumer reads its input, or None when the consumer
|
||||
runs in the next layer and ``edge.need`` is the rows this layer hands on:
|
||||
then only the producer's half runs here. ``fusions`` are the fused kernels
|
||||
the consumer tries first (FusedMlpInput into an FFN, the attention input's
|
||||
entries into an attention); ``enters_stack`` for the edge into the layer
|
||||
stack's first attention. The consumer's half reads only this edge's
|
||||
declarations, never what the producer chose for a batch; a sum left for a
|
||||
batch arrives with the value."""
|
||||
if reads is None:
|
||||
if edge.need.layout != edge.residual_to:
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
output_move=None if returns_over_dp else output_move,
|
||||
output_move_completes_sum=completes_sum,
|
||||
)
|
||||
if reads is InputRead.FFN:
|
||||
input_step, fused = _select_ffn_input(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
read=edge.need.read,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(edge, prepare=input_step, fused=fused)
|
||||
layer_input = None
|
||||
if edge.produced.always_leaves:
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which the
|
||||
# attention takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
edge.produced.group is not SumGroup.TP
|
||||
or edge.residual.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in edge.need.gathers_itself
|
||||
or edge.residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
|
||||
):
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
"""The consumer's half of ``edge``, chosen from the edge's declarations and
|
||||
the producer's update: what a value carries for a batch (a sum or a handoff
|
||||
its producer left) is completed first, trying ``carried_fusions``; a sum
|
||||
the producer always leaves as the layouts say, trying ``fusions``. Fused
|
||||
kernels run only for a plain add and a read that is the residual's norm
|
||||
and leaves the residual as it is; orders that add the residual before the
|
||||
sum completes only for a plain add. ``cp_moves`` for an edge that gathers
|
||||
over attention CP; ``enters_stack`` for the edge into the layer stack's
|
||||
first stage. The steps read only this edge's declarations, never what the
|
||||
producer chose for a batch."""
|
||||
update = edge.produced.update
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
plain = update.adds_plainly and edge.need.read.norms_plainly
|
||||
step, fused, input_move = _select_input_steps(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
update=update,
|
||||
fusions=fusions if plain else (),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
prepare=partial(
|
||||
_attention_input_step,
|
||||
layer_input=layer_input,
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
fusions=(
|
||||
fusions
|
||||
if edge.produced.update.adds_plainly and edge.need.read.norms_plainly
|
||||
else ()
|
||||
),
|
||||
enters_stack=enters_stack,
|
||||
read=edge.need.read,
|
||||
update=edge.produced.update,
|
||||
_consumer_step,
|
||||
step=step,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
owes_by_construction=edge.produced.always_leaves,
|
||||
),
|
||||
input_move=_select_attention_input_move(edge.residual_to, edge.need),
|
||||
input_move=input_move,
|
||||
fused=fused,
|
||||
)
|
||||
|
||||
|
||||
def make_output_boundary(
|
||||
edge: EdgeDecl, *, cp_moves: Optional[CpMoves] = None
|
||||
) -> Boundary:
|
||||
"""The producer's half of ``edge``, out of a layer's last stage onto the
|
||||
rows the layer hands on (``edge.need``), whose consumer runs in the next
|
||||
layer: the postprocess that moves the output there. ``cp_moves`` for an
|
||||
edge that returns across attention CP."""
|
||||
if edge.need.layout != edge.residual_to:
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
output_move=None if returns_over_dp else output_move,
|
||||
output_move_completes_sum=completes_sum,
|
||||
)
|
||||
|
||||
|
||||
@@ -760,18 +691,18 @@ def _select_boundary_steps(
|
||||
input tries ``attention_fusions``, and the layer stack's first layer
|
||||
(``enters_stack``) starts its residual there."""
|
||||
edges = decoder_layer_edges(sides)
|
||||
out_of_ffn = make_boundary(edges.out_of_ffn, reads=None, cp_moves=cp_moves)
|
||||
out_of_ffn = make_output_boundary(edges.out_of_ffn, cp_moves=cp_moves)
|
||||
into_ffn = make_boundary(
|
||||
edges.into_ffn,
|
||||
reads=InputRead.FFN,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
)
|
||||
into_attention = make_boundary(
|
||||
edges.into_attention,
|
||||
reads=InputRead.ATTENTION,
|
||||
fusions=attention_fusions,
|
||||
carried_fusions=attention_fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return BoundarySteps(
|
||||
@@ -784,6 +715,7 @@ def _select_boundary_steps(
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
input_rows=into_ffn.input_rows,
|
||||
input_move=into_ffn.input_move,
|
||||
fused=into_ffn.fused,
|
||||
),
|
||||
ffn_output=edges.out_of_ffn.produced,
|
||||
@@ -793,69 +725,33 @@ def _select_boundary_steps(
|
||||
)
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
gathers them itself."""
|
||||
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
|
||||
if not need.layout.sharded <= rows.sharded or gathered not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_TP_SCATTER},
|
||||
):
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
if gathered:
|
||||
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
|
||||
def _select_ffn_input(
|
||||
def _select_input_steps(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
residual: Layout,
|
||||
residual_to: Layout,
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
update: StageUpdate,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
the rows it has while the FFN runs, write the output into it and read the
|
||||
FFN input, and bring the rows to what the FFN's group needs: a gather over
|
||||
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
|
||||
only when nothing is gathered or sliced, and only if it completes the sum
|
||||
the attention output owes. A write-back that is not a plain add runs only
|
||||
after the sum completes."""
|
||||
if produced.leaves_for_next_layer and produced.group not in (
|
||||
None,
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
# A producer that leaves its sum only for some batches (an FFN before
|
||||
# this one) hands that sum on with the value: complete it, then the
|
||||
# input is a complete output.
|
||||
step, fused = _select_ffn_input(
|
||||
StageOutput(produced.layout),
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=need,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
read=read,
|
||||
update=update,
|
||||
)
|
||||
return partial(_mlp_input_completing_owed, step=step), fused
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
if owes_attention_tp != produced.always_leaves or produced.group not in (
|
||||
None,
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
force_layernorm_before_gather: bool,
|
||||
residual_joins_sum: bool,
|
||||
cp_moves: Optional[CpMoves],
|
||||
enters_stack: bool,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...], Optional[Callable]]:
|
||||
"""The steps into a stage for a value that is complete or owes the sum its
|
||||
producer always leaves, the fused kernels they try first, and the move
|
||||
after them: complete that sum, move the residual to the rows it has while
|
||||
the stage runs, write the output into it with ``update`` and read the input,
|
||||
and bring the input onto the rows the stage needs: a gather over attention
|
||||
DP or CP, each rank's own slice, or after the read a gather over attention
|
||||
TP. A kernel in ``fusions`` is tried only when nothing is gathered or
|
||||
sliced, and only if it completes the sum the input owes. An update that is
|
||||
not a plain add runs only after the sum completes."""
|
||||
read = need.read
|
||||
# What the input owes by construction; a sum the producer leaves only for
|
||||
# some batches comes with the value and is completed before these steps.
|
||||
owes = produced.group if produced.always_leaves else None
|
||||
if produced.always_leaves and owes is None:
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
@@ -866,44 +762,89 @@ def _select_ffn_input(
|
||||
if (
|
||||
sliced != {TokenAxis.ATTN_TP_SCATTER}
|
||||
or gathered
|
||||
or owes not in (None, SumGroup.ATTN_TP)
|
||||
or residual_to != need.layout
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
|
||||
_mlp_input_scatter if owes is SumGroup.ATTN_TP else _mlp_input_slice,
|
||||
scatters_residual=residual != residual_to,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# The residual stays on each rank's slice while the FFN takes the
|
||||
# attention's rows (MHC on an input-scattered batch).
|
||||
if gathered or not owes_attention_tp or residual != residual_to:
|
||||
if residual == residual_to:
|
||||
# The residual stays on each rank's slice while the stage takes the
|
||||
# rows around it (MHC on an input-scattered batch).
|
||||
if gathered or owes is not SumGroup.ATTN_TP:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_on_residual_shard, read=read, update=update),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which
|
||||
# the stage takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
owes is not SumGroup.TP
|
||||
or residual.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in need.gathers_itself
|
||||
or residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_on_residual_shard, read=read, update=update),
|
||||
partial(
|
||||
_read_input,
|
||||
layer_input=tp_reduce_scatter,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
on_chunk, fused = _select_ffn_input(
|
||||
on_chunk, fused, _ = _select_input_steps(
|
||||
produced,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=StageInput(produced.layout),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
read=read,
|
||||
need=StageInput(produced.layout, read=read),
|
||||
update=update,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused, None
|
||||
if gathered == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# A complete input on each rank's slice, gathered over attention TP
|
||||
# once it is read.
|
||||
if (
|
||||
owes is not None
|
||||
or residual != residual_to
|
||||
or residual_to != produced.layout
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_read_input,
|
||||
layer_input=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
CommunicateSimpleFn._scattered_to_tp_attn_full,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
or gathered
|
||||
@@ -919,30 +860,48 @@ def _select_ffn_input(
|
||||
# A residual arriving on each rank's slice is gathered back first.
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return partial(_mlp_input_norm, read=read, update=update), ()
|
||||
if owes is None:
|
||||
if gathers_residual:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_read_input,
|
||||
layer_input=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if not update.adds_plainly:
|
||||
if owes is not SumGroup.ATTN_TP or not update.adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_residual_into_sum, read=read), ()
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
fused = (
|
||||
tuple(f for f in fusions if f.completes is produced.group)
|
||||
if update.adds_plainly and read.norms_plainly
|
||||
else ()
|
||||
)
|
||||
return partial(_mlp_input_residual_into_sum, read=read), (), None
|
||||
# A sum over TP completes only on rows every TP rank holds: the TP group
|
||||
# spans attention DP and CP.
|
||||
if owes is not SumGroup.ATTN_TP and (
|
||||
owes is not SumGroup.TP or produced.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
fused = tuple(f for f in fusions if f.completes is owes)
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
group=owes,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
fused,
|
||||
None,
|
||||
)
|
||||
if owes not in (None, SumGroup.ATTN_TP):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
owes_attention_tp = owes is SumGroup.ATTN_TP
|
||||
# The partial order adds the residual on attention-TP rank 0 before the DP
|
||||
# gather's collective completes that sum, which only a plain residual add
|
||||
# allows.
|
||||
@@ -963,6 +922,7 @@ def _select_ffn_input(
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
partial(
|
||||
@@ -974,6 +934,7 @@ def _select_ffn_input(
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -31,14 +31,15 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
DecoderLayerSides,
|
||||
FusedMlpInput,
|
||||
InputRead,
|
||||
LayerStage,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
_cp_moves,
|
||||
_select_boundary_steps,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
scattered_residual_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
with_residual,
|
||||
@@ -419,30 +420,26 @@ class LayerCommunicator:
|
||||
self._declared = None
|
||||
self._cp_steps = self._input_scattered_steps = self._sp_steps = None
|
||||
into_edge, out_edge = stage.edges
|
||||
reads_ffn = stage.reads is InputRead.FFN
|
||||
is_ffn = stage.kind is StageKind.FFN
|
||||
self.stage_edges = stage.edges
|
||||
into = make_boundary(
|
||||
into_edge,
|
||||
reads=stage.reads,
|
||||
fusions=(
|
||||
self._select_mlp_input_fusions()
|
||||
if reads_ffn
|
||||
else self._attn_input_fusions
|
||||
),
|
||||
fusions=self._select_mlp_input_fusions() if is_ffn else (),
|
||||
carried_fusions=() if is_ffn else self._attn_input_fusions,
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
enters_stack=stage.enters_stack,
|
||||
)
|
||||
out = make_boundary(out_edge, reads=None)
|
||||
out = make_output_boundary(out_edge)
|
||||
entry = StageEntry(
|
||||
prepare=into.prepare,
|
||||
input_rows=into.input_rows,
|
||||
input_move=None if reads_ffn else into.input_move,
|
||||
handoff=None if reads_ffn else _hand_qkv_hook_its_input,
|
||||
input_move=into.input_move,
|
||||
handoff=None if is_ffn else _hand_qkv_hook_its_input,
|
||||
fused=into.fused,
|
||||
)
|
||||
self._steps = BoundarySteps(
|
||||
attention=None if reads_ffn else entry,
|
||||
ffn=entry if reads_ffn else None,
|
||||
attention=None if is_ffn else entry,
|
||||
ffn=entry if is_ffn else None,
|
||||
ffn_output=out_edge.produced,
|
||||
ffn_output_move=out.output_move,
|
||||
ffn_output_move_completes_sum=out.output_move_completes_sum,
|
||||
@@ -662,10 +659,15 @@ class LayerCommunicator:
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
gathered_last_layer_output = self.attn.entry(forward_batch).input_move(
|
||||
hidden_states=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
move = self.attn.entry(forward_batch).input_move
|
||||
gathered_last_layer_output = (
|
||||
residual
|
||||
if move is None
|
||||
else move(
|
||||
hidden_states=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
if (
|
||||
gathered_last_layer_output is residual
|
||||
@@ -762,11 +764,9 @@ class LayerCommunicator:
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly: what arrives is written in as this layer's FFN writes its own."""
|
||||
if not (
|
||||
self._residual.ffn_update.adds_plainly
|
||||
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
plainly; the boundary tries them only when the update it writes in is a
|
||||
plain add."""
|
||||
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
self._attn_input_fuses_quant = (
|
||||
self.enable_fused_ar_quant
|
||||
@@ -821,11 +821,9 @@ class LayerCommunicator:
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly."""
|
||||
if not (
|
||||
self._residual.attention_update.adds_plainly
|
||||
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
order they are tried. They add the residual plainly; the boundary tries
|
||||
them only when the update it writes in is a plain add."""
|
||||
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
||||
return ()
|
||||
return (
|
||||
FusedMlpInput(
|
||||
|
||||
@@ -36,11 +36,16 @@ from sglang.srt.layers.communicator.adapters.attention import (
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
@@ -249,19 +254,26 @@ def _mlp_input_without_dp(
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Complete the sum the input owes over ``group`` on the rows it is on,
|
||||
unless one of ``fusions`` does it with the residual add and the norm, then
|
||||
write it into the residual and read the input."""
|
||||
if gathers_residual:
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
if group is SumGroup.ATTN_TP:
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
else:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
@@ -381,33 +393,88 @@ def _tp_all_gather_scattered_rows(
|
||||
return output
|
||||
|
||||
|
||||
def _mlp_input_completing_owed(
|
||||
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
|
||||
residual: torch.Tensor,
|
||||
def _consumer_step(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
norm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
step: Callable,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
owes_by_construction: bool = False,
|
||||
**call,
|
||||
):
|
||||
"""An FFN input whose producer may have left its sum: complete what the
|
||||
value owes, then run ``step`` on the complete output."""
|
||||
return step(
|
||||
reduce_output(hidden_states), residual, forward_batch, layernorm, context
|
||||
)
|
||||
"""A boundary's half into a stage: complete what the value carries, a sum
|
||||
or a handoff its producer left for this batch, then run ``step`` on the
|
||||
complete value. Under attention DP the reduction back to this rank's tokens
|
||||
comes first; otherwise one of ``carried_fusions`` may complete it with the
|
||||
residual add and the read, and an empty batch has nothing to sum. An input
|
||||
that owes its sum by construction (``owes_by_construction``) is that sum's
|
||||
only carrier. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if owes_by_construction:
|
||||
raise RuntimeError(
|
||||
f"an input that owes its sum by construction arrived as "
|
||||
f"{type(owed).__name__}"
|
||||
)
|
||||
if (
|
||||
isinstance(owed, UnreducedOutput)
|
||||
and owed.reduce_and_redistribute is not None
|
||||
):
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
for fused in carried_fusions:
|
||||
result = fused(
|
||||
owed, residual, forward_batch, call.get("post_residual_addition")
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
return step(hidden_states, residual, forward_batch, norm, context, **call)
|
||||
|
||||
|
||||
def _mlp_input_norm(
|
||||
def _read_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
norm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
layer_input: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
"""Complete what the input owes by construction (``layer_input``), then
|
||||
write the previous stage's output into the residual and read the stage's
|
||||
input with ``norm``. The layer stack's first stage (``enters_stack``)
|
||||
starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
if enters:
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
return read.update_and_read(
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import Optional
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
InputRead,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerStage,
|
||||
@@ -13,6 +12,7 @@ from sglang.srt.layers.communicator import (
|
||||
ScatterMode,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -29,12 +29,13 @@ def is_attn_layer(layer_type: str) -> bool:
|
||||
return layer_type in ATTN_LAYERS
|
||||
|
||||
|
||||
def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]:
|
||||
"""How the stage at ``layer_idx`` reads its input: a Mamba or attention
|
||||
mixer like an attention, an MLP or MoE like an FFN; None past either end."""
|
||||
def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]:
|
||||
"""Which stage of a decoder layer the stage at ``layer_idx`` stands for: a
|
||||
Mamba or attention mixer the attention, an MLP or MoE the FFN; None past
|
||||
either end."""
|
||||
if not 0 <= layer_idx < len(pattern):
|
||||
return None
|
||||
return InputRead.ATTENTION if is_attn_layer(pattern[layer_idx]) else InputRead.FFN
|
||||
return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN
|
||||
|
||||
|
||||
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
@@ -56,8 +57,8 @@ def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
always_leaves=owes and following is InputRead.FFN,
|
||||
leaves_for_next_layer=owes and following is InputRead.ATTENTION,
|
||||
always_leaves=owes and following is StageKind.FFN,
|
||||
leaves_for_next_layer=owes and following is StageKind.ATTENTION,
|
||||
),
|
||||
)
|
||||
sparse = pattern[layer_idx] == MOE
|
||||
@@ -75,7 +76,7 @@ def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
StageOutput(
|
||||
full,
|
||||
group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=following is InputRead.ATTENTION,
|
||||
leaves_for_next_layer=following is StageKind.ATTENTION,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
),
|
||||
@@ -89,7 +90,7 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
return LayerStage(
|
||||
reads=_stage_kind(pattern, layer_idx),
|
||||
kind=_stage_kind(pattern, layer_idx),
|
||||
edges=stage_edges(
|
||||
previous=(
|
||||
_stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None
|
||||
@@ -126,7 +127,7 @@ def make_layer_communicator(
|
||||
"""The communicator of a layer that is one stage: only its own norm, and
|
||||
boundaries built from the stages next to it in the pattern."""
|
||||
stage = layer_stage(pattern, layer_idx)
|
||||
for_attn = stage.reads is InputRead.ATTENTION
|
||||
for_attn = stage.kind is StageKind.ATTENTION
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=_build_layer_scatter_modes(
|
||||
pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1
|
||||
|
||||
Reference in New Issue
Block a user