[Refactor] Retire the layer facade and simplify boundary internals (#41554)

This commit is contained in:
Cheng Wan
2026-09-28 20:17:33 -07:00
committed by GitHub
parent 9b89b96230
commit 701213c867
101 changed files with 3081 additions and 6032 deletions
@@ -195,7 +195,10 @@ class Parallel(msgspec.Struct):
] = None
enable_attn_tp_input_scattered: A[
bool,
"Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.",
Arg(
help="Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.",
resolvable=True,
),
] = False
enable_shared_experts_attn_tp: A[
bool,
@@ -29,6 +29,27 @@ from sglang.srt.utils.common import parse_connector_type
logger = logging.getLogger(__name__)
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
"""Resolve the token-row modes the model's forward actually enters."""
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
longcat = model_type == "longcat_flash"
if nemotron and cfg.attn_cp_size > 1:
raise ValueError("Nemotron-H does not support --attn-cp-size > 1")
if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1:
raise ValueError(
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
)
if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered:
# Neither model enters the input-scattered attention scope. Preserve
# their existing ordinary execution and make the effective flag explicit.
logger.warning(
"Disabling input-scattered attention for %s: its forward does not enter that scope",
model_type,
)
return {"enable_attn_tp_input_scattered": False}
return {}
def handle_context_parallelism(server_args: Any):
# Through the registry, not a bare call: an out-of-tree replacement of
# `validate_prefill_cp_platform` registered at its own (earlier) pipeline
@@ -42,6 +63,13 @@ def handle_context_parallelism(server_args: Any):
model_config = model_config_of(server_args)
hf_config = model_config.hf_config
model_arch = hf_config.architectures[0]
declare_resolution(
server_args,
"boundary_parallelism",
**_boundary_parallelism_overrides(
cfg, model_config.hf_text_config.model_type
),
)
if (
cfg.enable_prefill_cp
and model_arch == "DeepseekV32ForCausalLM"
@@ -957,7 +957,9 @@ def model_forward_stages(
hidden_states, residual = stream.finish(hidden_states)
inputs["hidden_states"] = hidden_states
parts = _model_forward_tbo_split_inputs(
**inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows
**inputs,
residual=residual,
layer_input_rows=layers[0].attn_stage.incoming_residual_rows,
)
for part in parts:
part["hidden_states"], child_stream = ResidualStream.arrive(
@@ -992,89 +994,6 @@ def model_forward_stages(
return hidden_states
def model_forward_maybe_tbo(
layers,
enable_tbo: bool,
positions: torch.Tensor,
forward_batch: ForwardBatch,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
zero_allocator: Optional[BumpAllocator] = None,
):
inputs = dict(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
)
operations_strategy = OperationsStrategy.init_new_tbo(
layers, forward_batch.global_forward_mode
)
if enable_tbo:
return _model_forward_tbo(
inputs=inputs,
operations_strategy=operations_strategy,
layer_input_rows=layers[0].layer_communicator.input_rows,
)
else:
return _model_forward_non_tbo(inputs, operations_strategy)
def _model_forward_tbo(
inputs,
operations_strategy: OperationsStrategy,
layer_input_rows: Layout,
):
parent_batch = inputs["forward_batch"]
stream = inputs["residual"]
pending = stream.pending if isinstance(stream, ResidualStream) else None
inputs["hidden_states"], inputs["residual"] = finish_layer_stack(
inputs["hidden_states"], inputs["residual"], inputs["forward_batch"]
)
inputs_arr = _model_forward_tbo_split_inputs(
**inputs, layer_input_rows=layer_input_rows
)
original_hidden_states_len = inputs["hidden_states"].shape[0]
if isinstance(stream, ResidualStream):
for part in inputs_arr:
part["hidden_states"], part["residual"] = ResidualStream.arrive(
part["hidden_states"],
part["residual"],
pending.update if pending is not None else None,
)
part["forward_batch"].residual_stream = part["residual"]
del inputs, stream, pending
context = (
empty_context()
if _is_hip
else deep_gemm_wrapper.configure_deep_gemm_num_sms(
operations_strategy.deep_gemm_num_sms
)
)
with context:
outputs_arr = execute_overlapped_operations(
inputs_arr=inputs_arr,
operations_arr=[operations_strategy.operations] * 2,
delta_stages=[0, operations_strategy.tbo_delta_stages],
)
hidden_states, residual = _model_forward_tbo_merge_outputs(
*outputs_arr, original_hidden_states_len
)
parent_batch.residual_stream = (
residual if isinstance(residual, ResidualStream) else None
)
return hidden_states, residual
def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy):
outputs = execute_operations(inputs, operations_strategy.operations)
return outputs["hidden_states"], outputs["residual"]
def _model_forward_tbo_split_inputs(
hidden_states: torch.Tensor,
residual: torch.Tensor,
@@ -18,24 +18,25 @@ from sglang.srt.layers.communicator.adapters.attention import (
get_attn_tp_context,
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
EdgeDecl,
FusedMlpInput,
LayerStage,
StageDecl,
StageEntry,
StageInput,
StageKind,
StageOutput,
make_boundary,
make_output_boundary,
stage_edges,
tbo_split_moves,
)
from sglang.srt.layers.communicator.construction import (
BatchVariant,
StageEdges,
make_stage,
)
from sglang.srt.layers.communicator.contracts import (
EdgeDecl,
FusedMlpInput,
HandoffRows,
ProducerReduction,
StageDecl,
StageEntry,
StageInput,
StageKind,
StageOutput,
StageSteps,
)
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
from sglang.srt.layers.communicator.factories import (
@@ -55,8 +56,6 @@ from sglang.srt.layers.communicator.layout import (
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
move_rows,
tp_reduce_scatter,
)
@@ -78,9 +77,6 @@ from sglang.srt.layers.communicator.residual.mhc import (
)
__all__ = [
"LayerCommunicator",
"LayerFacts",
"MHCLayerCommunicator",
"declare_attn",
"declare_ffn",
"make_attn_stage",
@@ -88,17 +84,16 @@ __all__ = [
"make_stages",
"ADD",
"AttentionInputs",
"BoundarySteps",
"CommunicateSimpleFn",
"CommunicateSummableTensorPairFn",
"StageSteps",
"EdgeDecl",
"HandoffRows",
"ProducerReduction",
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"FfnCompletion",
"FfnExit",
"FusedMlpInput",
"HandoffOutput",
"LayerResidual",
"LayerStage",
"Layout",
"MHCState",
"MixerExit",
@@ -121,14 +116,7 @@ __all__ = [
"move_rows",
"reduce_output",
"sparse_moe_gathers_over_moe_cp",
"stage_edges",
"tbo_split_moves",
"token_axis_sizes",
"tp_reduce_scatter",
]
from sglang.srt.layers.communicator.layer import (
LayerCommunicator,
LayerFacts,
MHCLayerCommunicator,
)
@@ -135,11 +135,6 @@ class AttnTpContext:
def set_attn_inputs(self, attn_inputs: AttentionInputs):
get_forward().set("attn_inputs", attn_inputs)
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
attn_inputs = get_forward().attn_inputs
if attn_inputs is not None:
attn_inputs.hidden_states_local = hidden_states
def fetch_qkv_latent(self):
attn_inputs = get_forward().attn_inputs
assert attn_inputs is not None
@@ -15,7 +15,7 @@
from __future__ import annotations
from typing import Tuple
from typing import TYPE_CHECKING, Tuple
import torch
@@ -23,65 +23,63 @@ from sglang.srt.layers.communicator.ops import move_rows
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
if TYPE_CHECKING:
from sglang.srt.layers.communicator.construction import StagePlan
def branch_input(
self,
source: object,
plan: StagePlan,
source: StagePlan,
hidden_states: torch.Tensor,
residual: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> Tuple[torch.Tensor, ResidualStream]:
"""The FFN input and residual that ``source``'s boundary read for its
own FFN, for this layer's FFN, which branches from the same input: moved
to the rows this FFN needs and its residual's rows."""
rows, residual_rows, _ = source._branch_rows(forward_batch)
to, residual_to, _ = self._branch_rows(forward_batch)
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
if stream.pending is not None:
raise RuntimeError("a branch must start from a prepared stage input")
residual = stream.residual
to, residual_to, _ = plan._branch_rows(forward_batch)
if stream.pending is not None:
raise RuntimeError("a branch must start from a prepared stage input")
residual = stream.residual
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
return (
hidden_states,
ResidualStream(residual) if stream is not None else residual,
ResidualStream(residual),
)
def branch_output(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
"""This layer's complete FFN output as a branch's contribution, which
adds to the layer's output without writing the residual: moved to the
rows the layer hands on."""
rows, _, to = self._branch_rows(forward_batch)
rows, _, to = plan._branch_rows(forward_batch)
return move_rows(hidden_states, rows, to, forward_batch)
def merge_branch(
self,
plan: StagePlan,
contribution: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor,
source: object,
stream: ResidualStream,
source: StagePlan,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> Tuple[torch.Tensor, ResidualStream]:
"""A contribution from ``branch_output`` summed with what ``source``'s
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
this layer hands on."""
_, _, rows = source._branch_rows(forward_batch)
_, _, to = self._branch_rows(forward_batch)
stream = residual if isinstance(residual, ResidualStream) else None
update = None
if stream is not None:
stream.check(hidden_states)
update = stream.pending.update
hidden_states, residual = stream.finish(hidden_states)
_, _, to = plan._branch_rows(forward_batch)
stream.check(hidden_states)
if stream.pending is None:
raise RuntimeError("branch merge requires a pending producer contribution")
update = stream.pending.update
hidden_states, residual = stream.finish(hidden_states)
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, rows, to, forward_batch)
output = contribution + hidden_states
if stream is not None:
stream.write(residual)
return stream.leave(output, update), stream
return output, residual
stream.write(residual)
return stream.leave(output, update), stream
@@ -0,0 +1,26 @@
"""Row requirements at the entrance to two-batch overlap."""
from sglang.srt.layers.communicator.contracts import HandoffRows
from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp
from sglang.srt.runtime_context import get_exec
def tbo_handoff(sparse, next_sparse):
"""Record a potential split between a dense layer and a sparse layer."""
return HandoffRows.TBO_SPLIT if not sparse and next_sparse else None
def resolve_handoff_rows(requirement):
"""Bind a handoff requirement with the construction-time parallel config.
Dense-local output must reach attention rows before the TBO split.
Moving that gather after the split changes padding and collective sizes.
"""
if requirement is HandoffRows.TBO_SPLIT:
return (
HandoffRows.ATTENTION
if enable_moe_dense_fully_dp()
and get_exec().overlap.enable_two_batch_overlap
else None
)
return requirement
+54 -633
View File
@@ -14,14 +14,18 @@
"""What each stage produces and needs, and the boundary steps chosen from those
declarations."""
from enum import Enum, auto
from functools import partial
from typing import Callable, FrozenSet, Mapping, Optional, Tuple
from typing import Callable, Optional, Tuple
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.contracts import (
CpMoves,
EdgeDecl,
FusedMlpInput,
StageInput,
StageOutput,
)
from sglang.srt.layers.communicator.layout import (
Layout,
SumGroup,
@@ -32,11 +36,22 @@ from sglang.srt.layers.communicator.layout import (
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
gather_attention_tp,
identity_output,
move_rows,
output_on_residual_shard,
reduce_scatter_over_cp,
scatter_moe_cp_output,
scatter_output,
take_back_attention_cp_shard,
take_back_cp_shard,
tp_reduce_scatter,
tp_slice,
update_and_gather,
)
from sglang.srt.layers.communicator.prepare import (
_consumer_step,
_dispatch_consumer,
_hand_qkv_hook_its_input,
_mlp_input_dp_partial,
_mlp_input_dp_replicate,
_mlp_input_gather_attention_cp,
@@ -47,445 +62,10 @@ from sglang.srt.layers.communicator.ops import (
_mlp_input_slice,
_mlp_input_without_dp,
_read_input,
move_rows,
tp_reduce_scatter,
tp_slice,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import (
LayerResidual,
StageRead,
StageUpdate,
)
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
NORM_QUANT_READ,
NORM_READ,
)
from sglang.srt.runtime_context import get_parallel
class StageInput(msgspec.Struct, frozen=True):
"""The rows a stage's consumer needs: sharded over the token axes its
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
# How the consumer reads its input from the residual.
read: StageRead = NORM_READ
class StageOutput(msgspec.Struct, frozen=True):
"""What a stage's producer hands the boundary after it, fixed at
construction: its rows, the group its output is summed over, and when it
leaves that sum to the boundary instead of completing it.
What one output actually owes is carried with the value: a plain
tensor handed across layers is complete, an ``UnreducedOutput`` names what
is left. A raw compute output entering a boundary is read with this
declaration and the boundary's decision for the batch."""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The producer never reduces its output, e.g. a row-parallel projection
# built with reduce_results=False.
always_leaves: bool = False
# Otherwise it leaves the sum only when the boundary publishes the flag for
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
leaves_for_next_layer: bool = False
leaves_for_reduce_scatter: bool = False
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[StageUpdate] = ADD
transform: Optional[OutputTransform] = None
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class DecoderLayerSides(msgspec.Struct, frozen=True):
"""What a decoder layer declares about its attention, its FFN and the rows
between them; ``decoder_layer_edges`` turns it into the layer's
boundaries."""
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageDecl
ffn: StageDecl
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
# The group the layer's input arrives as a partial sum over, if any.
input_owes: Optional[SumGroup] = None
# Whether the residual is added into one rank's share of the attention
# output's sum before that sum completes, instead of after it.
residual_joins_attention_sum: bool = False
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
rows the residual is on before and after the boundary."""
produced: StageOutput
need: StageInput
residual: Layout
residual_to: Layout
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
update_capabilities: Tuple[bool, ...] = ()
class DecoderLayerEdges(msgspec.Struct, frozen=True):
"""A decoder layer's three boundaries: from the rows the previous layer
handed on into the attention, from the attention into the FFN, and from
the FFN onto the rows the layer hands on. The next layer runs the rest of
the last one."""
into_attention: EdgeDecl
into_ffn: EdgeDecl
out_of_ffn: EdgeDecl
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
"""The boundaries of a decoder layer, each with the declarations of its two
sides. What arrives at the layer is complete unless it owes a sum by
construction (``input_owes``); a sum the previous layer leaves for a batch
comes with the value."""
owes = sides.input_owes is not None
# Cross-layer arrival describes rows and owed work. The actual update
# arrives in the stream; this model family supplies only its capability.
arrived = StageOutput(
sides.input_rows,
group=sides.input_owes,
always_leaves=owes,
update=None,
)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
attention_rows = (
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
if owes
else sides.input_rows
)
return DecoderLayerEdges(
into_attention=EdgeDecl(
produced=arrived,
need=sides.attention.input,
residual=sides.input_rows,
residual_to=attention_rows,
update_capabilities=(sides.ffn.output.update.adds_plainly,),
),
into_ffn=EdgeDecl(
produced=sides.attention.output,
need=sides.ffn.input,
residual=attention_rows,
residual_to=sides.ffn_residual_rows,
residual_joins_sum=sides.residual_joins_attention_sum,
),
out_of_ffn=EdgeDecl(
produced=sides.ffn.output,
need=StageInput(sides.output_rows),
residual=sides.ffn_residual_rows,
residual_to=sides.output_rows,
),
)
def with_residual(
sides: DecoderLayerSides, residual: LayerResidual
) -> DecoderLayerSides:
"""``sides`` with the reads and updates the layer declares for its
attention and its FFN."""
replace = msgspec.structs.replace
return replace(
sides,
attention=StageDecl(
input=replace(sides.attention.input, read=residual.attention_read),
output=replace(sides.attention.output, update=residual.attention_update),
),
ffn=StageDecl(
input=replace(sides.ffn.input, read=residual.ffn_read),
output=replace(sides.ffn.output, update=residual.ffn_update),
),
)
def stage_edges(
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
) -> Tuple[EdgeDecl, EdgeDecl]:
"""The two boundaries of a layer that is one stage of a sequence of stages:
into it from the previous stage's output (None at the start of the layer
stack) and out of it onto ``rows``, the rows every layer hands on and the
residual is on between stages. The previous output arrives on those rows: a
stage whose output is elsewhere (an FFN on the TP group) moves it back there
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
adds_plainly = previous.update.adds_plainly if previous is not None else True
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
update=None,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows, update=None)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
EdgeDecl(
produced=arrived,
need=stage.input,
residual=rows,
residual_to=during,
update_capabilities=(adds_plainly,),
),
EdgeDecl(
produced=stage.output,
need=StageInput(rows),
residual=during,
residual_to=rows,
),
)
def sequence_parallel_layer_sides(
*, axis_sizes: Mapping[TokenAxis, int]
) -> DecoderLayerSides:
"""A decoder layer while a LayerNorm SP region is active. Its linears
all-gather their input and reduce-scatter their output themselves, so the
layer's rows, its residual and both outputs stay on each TP rank's slice."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out.
attention=StageDecl(
input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
output=StageOutput(local),
),
ffn=StageDecl(
input=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
),
output=StageOutput(local),
),
ffn_residual_rows=local,
output_rows=local,
)
def scattered_residual_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
is_first_layer: bool,
is_last_layer: bool,
leaves_for_reduce_scatter: bool,
) -> DecoderLayerSides:
"""A decoder layer on an input-scattered batch whose residual stays on
each attention-TP rank's slice (MHC). The attention and the FFN compute on
the full rows. The attention takes the slice and gathers it itself (its
QKV hook, or the handoff before an attention that needs the rows); the
boundary brings the FFN its rows and moves both outputs back onto the
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
is the embedding's partial sum on the full rows; the last layer hands on
the full rows again."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
),
ffn=StageDecl(
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
),
),
ffn_residual_rows=local,
output_rows=attention if is_last_layer else local,
input_owes=SumGroup.TP if is_first_layer else None,
)
def input_scattered_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
hands_on_partial: bool,
) -> DecoderLayerSides:
"""A decoder layer on a batch whose attention input is scattered over
attention TP. Each layer's input is a partial sum over TP on the full rows:
the embedding and every FFN before it leave that sum, and a reduce-scatter
completes it onto each rank's slice, which the attention gathers itself.
The residual comes back to the full rows inside the attention output's sum."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
return DecoderLayerSides(
input_rows=attention,
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=NORM_QUANT_READ,
),
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
),
ffn=StageDecl(
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next input completes the sum; the last FFN completes its own.
output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
),
),
ffn_residual_rows=attention,
output_rows=attention,
input_owes=SumGroup.TP,
residual_joins_attention_sum=True,
)
def decoder_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_on_local_rows: bool,
previous_on_local_rows: bool,
is_last_layer: bool,
attention_gathers_local_rows: bool,
ffn_group: SumGroup,
leaves_for_next_layer: bool,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
hands_on_attention_rows: bool = False,
ffn_shards_over_cp: bool = False,
) -> DecoderLayerSides:
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine, or a dense MLP on every rank).
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
computes each CP shard on its own ranks: its rows stay sharded over CP.
A layer on local rows hands on the attention's rows when it is the last, or
with ``hands_on_attention_rows``."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
# Each attention-TP rank's own slice of those rows.
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = (
local
if ffn_on_local_rows
else Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
axis_sizes=axis_sizes,
)
)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
input_rows=local if previous_on_local_rows else attention,
attention=StageDecl(
input=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
read=NORM_QUANT_READ,
),
# The output projection leaves the attention-TP sum to prepare_mlp.
output=StageOutput(
attention,
group=SumGroup.ATTN_TP if attention_tp else None,
always_leaves=attention_tp,
),
),
ffn=StageDecl(
input=StageInput(ffn),
output=StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
group=ffn_group,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
),
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=(
local
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
else attention
),
)
class FusedMlpInput(msgspec.Struct, frozen=True):
"""A kernel that completes the sum the attention output owes together with
the residual add and the post-attention norm, in that order.
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
the residual, or None when it does not take the batch; it returns None only
before touching its inputs or starting a collective."""
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
preserves_residual: Optional[Callable] = None
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
"""The moves around the two-batch-overlap split, which cuts the attention's
rows: from the rows the first overlapped layer takes to the attention's,
@@ -493,30 +73,16 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
)
pair = CommunicateSummableTensorPairFn
if layer_input_rows == attention:
return pair._trivial, pair._trivial
return identity_output, identity_output
if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice: write the residual in and gather over attention
# TP, then take the slice of each half.
return pair._gather, pair._scatter
return update_and_gather, scatter_output
raise NotImplementedError(f"{layer_input_rows=}")
class CpMoves(msgspec.Struct, frozen=True):
"""How a CP extend's rows reach an FFN that needs all of them and come back,
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
after each rank has completed its own block, ``take_back`` returns this
rank's block of a complete output, and ``reduce_scatter``, where there is
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
returns the block in the same collective."""
gather: Callable
take_back: Callable
reduce_scatter: Optional[Callable] = None
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
def _cp_moves() -> CpMoves:
"""DSA and MLA CP gather equal shards over the attention-CP group and can
complete a sum over it. GQA prefill CP gathers blocks padded to the longest
@@ -524,85 +90,16 @@ def _cp_moves() -> CpMoves:
if _gathers_over_attention_cp():
return CpMoves(
gather=_mlp_input_gather_attention_cp,
take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard,
reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp,
take_back=take_back_attention_cp_shard,
reduce_scatter=reduce_scatter_over_cp,
reduce_scatter_group=lambda: get_parallel().attn_cp_group,
)
return CpMoves(
gather=_mlp_input_gather_moe_cp,
take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
take_back=scatter_moe_cp_output,
)
class StageEntry(msgspec.Struct, frozen=True):
"""The boundary into one of a layer's stages, as the layer runs it."""
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
# (hidden_states, residual, forward_batch, norm, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
handoff: Optional[Callable] = None
# The fused kernels prepare tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
preserves_residual: Optional[Callable] = None
capture_preserves_residual: Optional[Callable] = None
input_sum: Optional[SumGroup] = None
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into each of its
stages, and the last stage's output on to the rows the layer hands on."""
# The boundary into the layer's attention and into its FFN; None for a
# stage a layer that is one stage does not have.
attention: Optional[StageEntry]
ffn: Optional[StageEntry]
# What the FFN exit reads: the FFN output's group and what it may leave.
ffn_output: StageOutput
# The postprocess that moves the FFN output on; None when it goes back over
# attention DP, whose step the FFN exit and postprocess choose per batch.
ffn_output_move: Optional[Callable]
# Whether the next layer's input can take the FFN's sum.
ffn_sum_is_movable: bool
# Whether ffn_output_move also completes the sum the FFN leaves.
ffn_output_move_completes_sum: bool = False
@property
def returns_over_dp(self) -> bool:
return self.ffn_output_move is None
class StageKind(Enum):
"""Which of a decoder layer's two stages a layer that is one stage takes
the place of: its norm, its read and update, its fused kernels and its
entry in the layer's steps."""
ATTENTION = auto()
FFN = auto()
class LayerStage(msgspec.Struct, frozen=True):
"""A layer that is one stage of a sequence of stages, each an
attention-like mixer or an FFN: which of the two it is, its two boundaries
(into it, and out of it onto the rows every layer hands on, as
``stage_edges`` gives them), and whether the layer stack
starts at it."""
kind: StageKind
edges: Tuple[EdgeDecl, EdgeDecl]
enters_stack: bool = False
class Boundary(msgspec.Struct, frozen=True):
"""The steps one layer runs at one boundary, chosen from both sides'
declarations. A layer runs the consumer's half of a boundary into one of
@@ -615,24 +112,20 @@ class Boundary(msgspec.Struct, frozen=True):
prepare: Optional[Callable] = None
# The move onto the consumer's rows after prepare; None when there is none.
input_move: Optional[Callable] = None
# The fused kernels prepare tries first on a sum the input always owes.
fused: Tuple["FusedMlpInput", ...] = ()
# The producer's half: the postprocess that moves the output onto the rows
# the layer hands on; None when it goes back over attention DP, whose step
# the FFN exit and postprocess choose per batch.
output_move: Optional[Callable] = None
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
returns_over_dp: bool = False
preserves_residual: Optional[Callable] = None
@property
def input_rows(self) -> Layout:
"""The rows the consumer is handed: what it needs, still sharded over
the axes it gathers itself as the rows its input is read on are."""
need = self.edge.need
return Layout(
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
)
return input_rows(self.edge)
@property
def capture_move_allocates(self) -> bool:
@@ -648,12 +141,18 @@ class Boundary(msgspec.Struct, frozen=True):
)
def input_rows(edge: EdgeDecl) -> Layout:
"""Consumer rows, retaining the axes it gathers inside its computation."""
return Layout(
edge.need.layout.sharded | (edge.residual_to.sharded & edge.need.gathers_itself)
)
def make_boundary(
edge: EdgeDecl,
*,
fusions: Tuple["FusedMlpInput", ...] = (),
carried_fusions: Tuple[Callable, ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> Boundary:
@@ -674,7 +173,6 @@ def make_boundary(
adds_plainly=capability,
fusions=fusions,
carried_fusions=carried_fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
@@ -692,9 +190,6 @@ def make_boundary(
_dispatch_consumer,
paths={capability: path.prepare for capability, path in paths.items()},
),
fused=tuple(
f for f in fusions if any(f in path.fused for path in paths.values())
),
)
@@ -704,7 +199,6 @@ def _bind_consumer(
adds_plainly: bool,
fusions: Tuple["FusedMlpInput", ...] = (),
carried_fusions: Tuple[Callable, ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> Boundary:
@@ -720,28 +214,26 @@ def _bind_consumer(
producer chose for a batch."""
# A fused kernel runs the add and the norm itself.
plain = adds_plainly and edge.need.read.norms_plainly
step, fused, input_move = _select_input_steps(
step, input_move = _select_input_steps(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
adds_plainly=adds_plainly,
fusions=fusions if plain else (),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
completed_step = None
if edge.produced.always_leaves:
completed_step, _, _ = _select_input_steps(
completed_step, _ = _select_input_steps(
msgspec.structs.replace(edge.produced, always_leaves=False),
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
adds_plainly=adds_plainly,
fusions=(),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=False,
cp_moves=cp_moves,
enters_stack=enters_stack,
@@ -750,14 +242,13 @@ def _bind_consumer(
# again (e.g. MHC expansion). Both alternatives are bound at construction.
written_step = None
if enters_stack:
written_step, _, _ = _select_input_steps(
written_step, _ = _select_input_steps(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
adds_plainly=adds_plainly,
fusions=fusions if plain else (),
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=False,
@@ -790,7 +281,6 @@ def _bind_consumer(
written_step=written_step,
),
input_move=input_move,
fused=fused,
)
@@ -818,73 +308,15 @@ def make_output_boundary(
residual=edge.residual,
to=edge.residual_to,
cp_moves=cp_moves,
update=edge.produced.update,
)
return Boundary(
edge,
output_move=None if returns_over_dp else output_move,
output_move=output_move,
returns_over_dp=returns_over_dp,
output_move_completes_sum=completes_sum,
)
def _select_boundary_steps(
sides: DecoderLayerSides,
*,
fusions: Tuple["FusedMlpInput", ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
attention_handoff: Callable = _hand_qkv_hook_its_input,
attention_fusions: Tuple[Callable, ...] = (),
enters_stack: bool = False,
) -> BoundarySteps:
"""The steps of a decoder layer: its three boundaries, each chosen from the
declarations of its two sides, around the layer's residual operations. Both
edges that cross attention CP take the same ``cp_moves``; the attention's
input tries ``attention_fusions``, and the layer stack's first layer
(``enters_stack``) starts its residual there."""
edges = decoder_layer_edges(sides)
out_of_ffn = make_output_boundary(edges.out_of_ffn, cp_moves=cp_moves)
into_ffn = make_boundary(
edges.into_ffn,
fusions=fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
)
into_attention = make_boundary(
edges.into_attention,
carried_fusions=attention_fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
return BoundarySteps(
attention=StageEntry(
prepare=into_attention.prepare,
input_rows=into_attention.input_rows,
input_move=into_attention.input_move,
handoff=attention_handoff,
capture_move=into_attention.capture_move,
capture_move_allocates=into_attention.capture_move_allocates,
input_sum=edges.into_attention.produced.group
if edges.into_attention.produced.always_leaves
else None,
),
ffn=StageEntry(
prepare=into_ffn.prepare,
input_rows=into_ffn.input_rows,
input_move=into_ffn.input_move,
fused=into_ffn.fused,
input_sum=edges.into_ffn.produced.group
if edges.into_ffn.produced.always_leaves
else None,
),
ffn_output=edges.out_of_ffn.produced,
ffn_output_move=out_of_ffn.output_move,
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
)
def _select_input_steps(
produced: StageOutput,
*,
@@ -893,11 +325,10 @@ def _select_input_steps(
need: StageInput,
adds_plainly: bool,
fusions: Tuple[FusedMlpInput, ...],
force_layernorm_before_gather: bool,
residual_joins_sum: bool,
cp_moves: Optional[CpMoves],
enters_stack: bool,
) -> Tuple[Callable, Tuple[FusedMlpInput, ...], Optional[Callable]]:
) -> Tuple[Callable, Optional[Callable]]:
"""The steps into a stage for a value that is complete or owes the sum its
producer always leaves, the fused kernels they try first, and the move
after them: complete that sum, move the residual to the rows it has while
@@ -933,7 +364,6 @@ def _select_input_steps(
scatters_residual=residual != residual_to,
read=read,
),
(),
None,
)
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
@@ -946,7 +376,6 @@ def _select_input_steps(
partial(
_mlp_input_on_residual_shard, read=read, reduces=owes is not None
),
(),
None,
)
# A reduce-scatter completes the TP sum onto each rank's slice, which
@@ -965,26 +394,24 @@ def _select_input_steps(
enters_stack=enters_stack,
read=read,
),
(),
None,
)
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the CP moves gather them.
if cp_moves is None:
raise NotImplementedError(f"{produced=} {need=}")
on_chunk, fused, _ = _select_input_steps(
on_chunk, _ = _select_input_steps(
produced,
residual=residual,
residual_to=residual_to,
need=StageInput(produced.layout, read=read),
adds_plainly=adds_plainly,
fusions=fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
residual_joins_sum=residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
return partial(cp_moves.gather, gather=on_chunk), fused, None
return (partial(cp_moves.gather, gather=on_chunk), None)
if gathered == {TokenAxis.ATTN_TP_SCATTER}:
# A complete input on each rank's slice, gathered over attention TP
# once it is read.
@@ -1001,8 +428,7 @@ def _select_input_steps(
enters_stack=enters_stack,
read=read,
),
(),
CommunicateSimpleFn._scattered_to_tp_attn_full,
gather_attention_tp,
)
if (
residual_to != produced.layout
@@ -1029,7 +455,6 @@ def _select_input_steps(
group=None,
read=read,
),
(),
None,
)
return (
@@ -1039,7 +464,6 @@ def _select_input_steps(
enters_stack=enters_stack,
read=read,
),
(),
None,
)
if gathers_residual and residual_joins_sum:
@@ -1047,7 +471,7 @@ def _select_input_steps(
# sum, so the all-reduce also brings the residual back to every row.
if owes is not SumGroup.ATTN_TP or not adds_plainly:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_residual_into_sum, read=read), (), None
return (partial(_mlp_input_residual_into_sum, read=read), None)
# A sum over TP completes only on rows every TP rank holds: the TP group
# spans attention DP and CP.
if owes is not SumGroup.ATTN_TP and (
@@ -1063,7 +487,6 @@ def _select_input_steps(
group=owes,
read=read,
),
fused,
None,
)
if owes not in (None, SumGroup.ATTN_TP):
@@ -1077,7 +500,7 @@ def _select_input_steps(
places_cp_shards = TokenAxis.ATTN_CP in gathered
if (
owes_attention_tp
and not force_layernorm_before_gather
and not read.before_gather
and adds_plainly
and read.norms_plainly
):
@@ -1088,7 +511,6 @@ def _select_input_steps(
places_cp_shards=places_cp_shards,
read=read,
),
(),
None,
)
return (
@@ -1099,7 +521,6 @@ def _select_input_steps(
places_cp_shards=places_cp_shards,
read=read,
),
(),
None,
)
@@ -1110,21 +531,21 @@ def _select_ffn_output_move(
residual: Layout,
to: Layout,
cp_moves: Optional[CpMoves] = None,
update: StageUpdate = ADD,
) -> Tuple[bool, Optional[Callable], bool]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
that step), or else the postprocess that moves it, None when there is none
to choose here; and whether that move also completes the sum the FFN
leaves."""
pair = CommunicateSummableTensorPairFn
update = produced.update
if produced.layout == residual:
if to == residual:
return False, pair._trivial, False
return False, identity_output, False
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: write the output
# into the residual, then gather over attention TP.
return False, partial(pair._gather, update=update), False
return False, partial(update_and_gather, update=update), False
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
@@ -1135,7 +556,7 @@ def _select_ffn_output_move(
return (
False,
partial(
pair._onto_residual_shard,
output_on_residual_shard,
sums=sums,
gathers_back=to != residual,
update=update,
@@ -1159,7 +580,7 @@ def _select_ffn_output_move(
return False, cp_moves.reduce_scatter, True
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
# This rank's CP shard, from where the DP gather put it.
return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False
return False, take_back_cp_shard, False
if returned != {TokenAxis.ATTN_DP}:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return True, None, False
@@ -14,22 +14,25 @@
"""Bind one stage to predeclared boundaries and reusable batch paths."""
from dataclasses import dataclass
from enum import Enum, auto
from functools import cached_property
from typing import Callable, Optional, Protocol, Tuple
from typing import Callable, Optional
from sglang.srt.environ import envs
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
EdgeDecl,
FusedMlpInput,
StageDecl,
StageEntry,
StageKind,
_cp_moves,
make_boundary,
make_output_boundary,
)
from sglang.srt.layers.communicator.contracts import (
BatchVariant,
CpMoves,
EdgeDecl,
ProducerReduction,
StageEntry,
StageKind,
StageSteps,
)
from sglang.srt.layers.communicator.exit import OutputBoundary
from sglang.srt.layers.communicator.fusions.allreduce import (
attention_fusions,
@@ -42,11 +45,14 @@ from sglang.srt.layers.communicator.layout import (
_generic_prefill_cp_shards_tokens,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.prepare import (
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
)
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
get_forward,
get_lora,
@@ -58,24 +64,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
class BoundaryFusions(Protocol):
"""The fused kernels a fusion backend gives a layer, bound to it and tried
before the layer's own: at its attention input (each takes what the previous
layer left, the residual, the batch and a post-residual addition), at its
FFN input, and at its FFN exit (each says what the exit does when its kernel
takes the batch)."""
def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ...
def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ...
requires_local_reduction: bool
def can_defer_finalize(
self, layer: "StagePlan", forward_batch: ForwardBatch
) -> bool: ...
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
"""A MoE layer under attention CP whose tokens the steps cannot bring
to it: under a prefill CP, one dispatched per DP shard under attention DP
@@ -118,13 +106,6 @@ def _input_can_be_scattered() -> bool:
)
class BatchVariant(Enum):
ORDINARY = auto()
CONTEXT_PARALLEL = auto()
INPUT_SCATTERED = auto()
SEQUENCE_PARALLEL = auto()
@dataclass(frozen=True)
class StageEdges:
"""One stage's incoming and outgoing declarations for one batch variant.
@@ -136,7 +117,7 @@ class StageEdges:
incoming: EdgeDecl
outgoing: EdgeDecl
handoff: Optional[Callable] = None
cp_moves: object = None
cp_moves: Optional[CpMoves] = None
def _requires_branch_input(*args, **kwargs):
@@ -155,26 +136,16 @@ class StagePlan:
enters_stack=False,
prepared_input=False,
terminal=False,
fixed_output=False,
is_sparse=False,
direct_handoff=False,
qkv_latent_func=None,
force_layernorm_before_dp_gather=False,
enable_fused_ar_quant=False,
fused_ar_quant_keep_bf16=False,
residual_in_hidden=False,
fusions=None,
):
self.kind = kind
self.norm = norm
self.variants = dict(variants)
self.is_first_layer = enters_stack
self.is_last_layer = terminal
self.fixed_output = fixed_output
self.is_sparse = is_sparse
self.enters_stack = enters_stack
self.terminal = terminal
self.direct_handoff = direct_handoff
self.qkv_latent_func = qkv_latent_func
self.enable_fused_ar_quant = enable_fused_ar_quant
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
self.residual_in_hidden = residual_in_hidden
self.fusions = fusions
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
@@ -183,8 +154,13 @@ class StagePlan:
get_lora().enable_lora
)
self._fusion_rows = None
carried = attention_fusions(self) if kind is StageKind.ATTENTION else ()
self._carried_fusions = carried
carried = (
attention_fusions(
self, next(iter(self.variants.values())).incoming.need.read
)
if kind is StageKind.ATTENTION
else ()
)
fused = ffn_fusions(self) if kind is StageKind.FFN else ()
self._paths = {}
for variant, edges in self.variants.items():
@@ -200,7 +176,6 @@ class StagePlan:
edges.incoming,
fusions=fused,
carried_fusions=carried,
force_layernorm_before_gather=force_layernorm_before_dp_gather,
cp_moves=edges.cp_moves,
enters_stack=enters_stack,
)
@@ -209,7 +184,6 @@ class StagePlan:
input_rows=into.input_rows,
input_move=into.input_move,
handoff=edges.handoff,
fused=into.fused,
capture_move=into.capture_move,
capture_move_allocates=into.capture_move_allocates,
preserves_residual=into.preserves_residual,
@@ -219,55 +193,54 @@ class StagePlan:
)
out = (
None
if fixed_output
if direct_handoff
else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves)
)
self._paths[variant] = BoundarySteps(
attention=entry if kind is StageKind.ATTENTION else None,
ffn=entry if kind is StageKind.FFN else None,
ffn_output=edges.outgoing.produced,
ffn_output_move=None if out is None else out.output_move,
ffn_output_move_completes_sum=False
self._paths[variant] = StageSteps(
entry=entry,
output=edges.outgoing.produced,
output_move=None if out is None else out.output_move,
returns_over_dp=out is not None and out.returns_over_dp,
output_move_completes_sum=False
if out is None
else out.output_move_completes_sum,
ffn_sum_is_movable=edges.outgoing.produced.group is not None,
)
self._steps = self._paths[BatchVariant.ORDINARY]
self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL)
self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED)
self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL)
@property
def input_rows(self):
def incoming_residual_rows(self):
return self.variants[BatchVariant.ORDINARY].incoming.residual
@property
def input_on_attention_tp_slices(self):
return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
return TokenAxis.ATTN_TP_SCATTER in self.incoming_residual_rows.sharded
def _variant(self, forward_batch):
if self._sp_steps is not None and get_forward().sp_active:
# The batch determines its rows; missing paths must not change them.
if get_forward().sp_active:
return BatchVariant.SEQUENCE_PARALLEL
if (
self._input_scattered_steps is not None
and get_attn_tp_context().input_scattered
):
if get_attn_tp_context().input_scattered:
return BatchVariant.INPUT_SCATTERED
if self._cp_steps is not None and _batch_shards_over_cp(forward_batch):
if _batch_shards_over_cp(forward_batch):
return BatchVariant.CONTEXT_PARALLEL
return BatchVariant.ORDINARY
def _batch_steps(self, forward_batch):
return self._paths[self._variant(forward_batch)]
variant = self._variant(forward_batch)
try:
return self._paths[variant]
except KeyError:
raise NotImplementedError(
f"no stage boundary path for the active {variant.name} batch"
) from None
def fusion_rows(self, forward_batch):
if self._fusion_rows is not None:
return self._fusion_rows[self._variant(forward_batch)]
entry = self._batch_steps(forward_batch)
return (entry.attention or entry.ffn).input_rows
return entry.entry.input_rows
def produced(self, kind, forward_batch):
return self._batch_steps(forward_batch).ffn_output
def produced(self, forward_batch):
return self._batch_steps(forward_batch).output
@cached_property
def output(self):
@@ -284,24 +257,47 @@ class StagePlan:
)
def make_stage(
declaration: StageDecl,
*,
kind: StageKind,
norm,
incoming: EdgeDecl,
outgoing: EdgeDecl,
variants=None,
handoff=None,
**options,
) -> StageCommunicator:
"""Build one real stage from its own declaration and adjacent edges.
def _bind_stage(declaration, norm, incoming, outgoing, **options):
if incoming.consumer != declaration or outgoing.producer != declaration:
raise ValueError("connections do not match the stage declaration")
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
variants = {}
for variant, edge in incoming.entries.items():
if (
declaration.update.at_producer
and TokenAxis.ATTN_CP
in edge.produced.layout.sharded - edge.need.layout.sharded
):
raise NotImplementedError("MHC with a gather over attention CP")
handoff = None
if declaration.kind is StageKind.ATTENTION:
handoff = (
_hand_scattered_input_to_attention
if variant is BatchVariant.INPUT_SCATTERED
and not declaration.update.adds_plainly
else _hand_qkv_hook_its_input
)
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
plan = StagePlan(
declaration.kind,
norm,
variants,
enters_stack=incoming.producer is None,
prepared_input=declaration.prepared_from is not None,
terminal=declaration.terminal,
direct_handoff=declaration.kind is StageKind.ATTENTION
and declaration.reduction is ProducerReduction.PARTIAL,
**options,
)
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
# Layout eligibility comes from the connected consumer, not a mutable
# link to its execution plan. Kernel binding remains consumer-owned.
from sglang.srt.layers.communicator.boundary import input_rows
The optional variants supply the same stage's CP/SP/input-scattered
declarations. Neither a paired stage nor its layer facts/norm is needed.
"""
if incoming.need != declaration.input or outgoing.produced != declaration.output:
raise ValueError("stage declaration disagrees with its boundaries")
paths = dict(variants or {})
paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff)
return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm)
plan._fusion_rows = {
v: input_rows(edge) for v, edge in outgoing.entries.items()
}
return StageCommunicator(plan, declaration=declaration)
@@ -0,0 +1,189 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Immutable stage declarations and bound input/output paths."""
from enum import Enum, auto
from typing import Callable, FrozenSet, Optional, Tuple
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
class ProducerReduction(Enum):
"""Whether compute always leaves its sum or follows the boundary scope."""
PARTIAL = auto()
SCOPED = auto()
# A replicated component is added after the sum inside compute.
LOCAL_TAIL = auto()
class HandoffRows(Enum):
"""A required layer/branch handoff, independent of compute's input rows."""
ATTENTION = auto()
TBO_SPLIT = auto()
class BatchVariant(Enum):
ORDINARY = auto()
CONTEXT_PARALLEL = auto()
INPUT_SCATTERED = auto()
SEQUENCE_PARALLEL = auto()
class StageInput(msgspec.Struct, frozen=True):
"""The rows a stage's consumer needs: sharded over the token axes its
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
# How the consumer reads its input from the residual.
read: StageRead = NORM_READ
class StageOutput(msgspec.Struct, frozen=True):
"""What a stage's producer hands the boundary after it, fixed at
construction: its rows, the group its output is summed over, and when it
leaves that sum to the boundary instead of completing it.
What one output actually owes is carried with the value: a plain
tensor handed across layers is complete, an ``UnreducedOutput`` names what
is left. A raw compute output entering a boundary is read with this
declaration and the boundary's decision for the batch."""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The producer never reduces its output, e.g. a row-parallel projection
# built with reduce_results=False.
always_leaves: bool = False
# Otherwise it leaves the sum only when the boundary publishes the flag for
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
leaves_for_next_layer: bool = False
leaves_for_reduce_scatter: bool = False
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[StageUpdate] = ADD
transform: Optional[OutputTransform] = None
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
rows the residual is on before and after the boundary."""
produced: StageOutput
need: StageInput
residual: Layout
residual_to: Layout
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
update_capabilities: Tuple[bool, ...] = ()
class FusedMlpInput(msgspec.Struct, frozen=True):
"""A kernel that completes the sum the attention output owes together with
the residual add and the post-attention norm, in that order.
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
the residual, or None when it does not take the batch; it returns None only
before touching its inputs or starting a collective."""
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
# Callable(residual, forward_batch): True guarantees this candidate is
# selected and does not mutate residual, including backend fallback.
preserves_residual: Optional[Callable] = None
class CpMoves(msgspec.Struct, frozen=True):
"""How a CP extend's rows reach an FFN that needs all of them and come back,
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
after each rank has completed its own block, ``take_back`` returns this
rank's block of a complete output, and ``reduce_scatter``, where there is
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
returns the block in the same collective."""
gather: Callable
take_back: Callable
reduce_scatter: Optional[Callable] = None
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
class StageEntry(msgspec.Struct, frozen=True):
"""The boundary into one of a layer's stages, as the layer runs it."""
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
# (hidden_states, residual, forward_batch, norm, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
handoff: Optional[Callable] = None
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
input_sum: Optional[SumGroup] = None
preserves_residual: Optional[Callable] = None
# Bound from a local successor at construction; no runtime plan lookup.
capture_preserves_residual: Optional[Callable] = None
class StageSteps(msgspec.Struct, frozen=True):
"""One batch variant of one stage: its input and output boundary paths."""
entry: StageEntry
output: StageOutput
# None means no fixed move. returns_over_dp selects batch-dependent DP transport.
output_move: Optional[Callable]
output_move_completes_sum: bool = False
returns_over_dp: bool = False
class StageKind(Enum):
"""Which of a decoder layer's two stages a layer that is one stage takes
the place of: its norm, its read and update, its fused kernels and its
entry in the layer's steps."""
ATTENTION = auto()
FFN = auto()
+91 -99
View File
@@ -16,7 +16,7 @@
from __future__ import annotations
from functools import partial
from typing import Callable, Optional, Tuple, Union
from typing import Callable, Optional, Tuple
import msgspec
import torch
@@ -51,11 +51,13 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
post_experts_reduction_group,
post_experts_sum_is_one_all_reduce,
should_use_dp_reduce_scatterv,
)
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
post_experts_reduction_group,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
get_exec,
@@ -95,32 +97,31 @@ class OutputBoundary:
def postprocess_layer(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
):
"""Move a complete output from the operation-scheduled producer path."""
stream = residual if isinstance(residual, ResidualStream) else None
steps = self.plan._batch_steps(forward_batch)
hidden_states, residual = self._complete_ffn_output_now(
hidden_states,
stream.residual if stream is not None else residual,
stream.residual,
forward_batch=forward_batch,
dp_step=None,
steps=steps,
)
return self._leave_ffn_output(
hidden_states,
residual,
stream,
self.plan._batch_steps(forward_batch).ffn_output.update,
steps.output.update,
)
@staticmethod
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
if stream is None:
return hidden_states, residual
stream.residual = residual
if update.at_producer:
return stream.write(hidden_states), stream
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
return stream.write(hidden_states)
return stream.leave(hidden_states, update, declared_sum=declared_sum)
@staticmethod
def _declared_ffn_sum(steps, skipped_reduction):
@@ -129,74 +130,69 @@ class OutputBoundary:
if (
skipped_reduction
and not steps.returns_over_dp
and not steps.ffn_output_move_completes_sum
and not steps.output_move_completes_sum
):
return SumGroup.TP
return None
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
def _local_token_move_can_go_to_next_layer(self, steps) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, when
the next layer's input also writes the output into the residual."""
steps = self.plan._batch_steps(forward_batch)
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
return steps.returns_over_dp and not steps.output.update.at_producer
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
self, forward_batch: ForwardBatch, steps
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings this layer's FFN output back to this
rank's tokens under attention DP; None when the base postprocess would
only scatter, or does not move tokens."""
steps = self.plan._batch_steps(forward_batch)
if not steps.returns_over_dp:
return None
return _reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
leaves_for_reduce_scatter=steps.output.leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=steps.output.leaves_for_reduce_scatterv,
)
def _ffn_leaves_sum_to_reduce_scatter(
self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
self, steps, dp_step: Optional[Callable]
) -> bool:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
steps = self.plan._batch_steps(forward_batch)
if dp_step is not None:
return True
if not steps.ffn_output.leaves_for_reduce_scatter:
if not steps.output.leaves_for_reduce_scatter:
return False
if steps.ffn_output_move_completes_sum:
if steps.output_move_completes_sum:
return True
return get_attn_tp_context().input_scattered and not self.plan.is_last_layer
return get_attn_tp_context().input_scattered and not self.plan.terminal
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
def ffn_reduction_group(self, steps) -> GroupCoordinator:
"""The group this layer's FFN output owes its sum over: the MoE output's
group on a sparse layer, the TP group a dense MLP reduces over."""
return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group)
return _sum_group(steps.output.group)
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
def _select_ffn_completion(
self, forward_batch: ForwardBatch, steps
) -> FfnCompletion:
"""Decide once, before the FFN runs, what it skips and what completes its
output: the next layer's input, or this layer's postprocess step."""
dp_step = self._postprocess_dp_step(forward_batch)
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
forward_batch, dp_step
)
dp_step = self._postprocess_dp_step(forward_batch, steps)
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(steps, dp_step)
complete_now = partial(
self._complete_ffn_output_now,
forward_batch=forward_batch,
dp_step=dp_step,
steps=steps,
)
steps = self.plan._batch_steps(forward_batch)
defer_moe_finalize = (
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
)
if not steps.ffn_output.leaves_for_next_layer and not (
self.plan.is_last_layer and defer_moe_finalize
if not steps.output.leaves_for_next_layer and not (
self.plan.terminal and defer_moe_finalize
):
return FfnCompletion(
defer_moe_finalize=False,
@@ -207,10 +203,10 @@ class OutputBoundary:
# Producers declare remaining work independently of the kernel chosen
# by the consumer. Every handoff also carries an unfused completion.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
)
if fuse_mlp_allreduce:
group = self.ffn_reduction_group(forward_batch)
group = self.ffn_reduction_group(steps)
if steps.returns_over_dp:
# Under attention DP the next layer also brings the sum back to
# this rank's tokens.
@@ -225,8 +221,8 @@ class OutputBoundary:
complete = partial(_leave_to_next_layer, wrap)
elif (
dp_step is not None
and not self.plan.is_last_layer
and self._local_token_move_can_go_to_next_layer(forward_batch)
and not self.plan.terminal
and self._local_token_move_can_go_to_next_layer(steps)
):
complete = partial(
_leave_to_next_layer,
@@ -253,51 +249,48 @@ class OutputBoundary:
*,
forward_batch: ForwardBatch,
dp_step: Optional[Callable],
steps,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step already
chosen: the move back to where the next layer reads the FFN output, then
the write-back into the residual for a layer that does it itself."""
steps = self.plan._batch_steps(forward_batch)
if steps.ffn_output.transform is not None:
hidden_states = steps.ffn_output.transform.apply(hidden_states)
if steps.output.transform is not None:
hidden_states = steps.output.transform.apply(hidden_states)
if steps.returns_over_dp:
hidden_states = _to_local_tokens(
dp_step or _redistribute_output, forward_batch, hidden_states
)
else:
hidden_states, residual = steps.ffn_output_move(
hidden_states, residual = steps.output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
update = steps.ffn_output.update
update = steps.output.update
if residual is not None and update.at_producer:
hidden_states = update.update(hidden_states, residual)
residual = None
return hidden_states, residual
def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit:
def mixer_exit(
self, forward_batch: ForwardBatch, *, stream: ResidualStream
) -> MixerExit:
"""Decide once whether this stage's mixer (an attention-like stage)
skips its output all-reduce. Use the result as a context manager around
the mixer, then call ``finish``."""
return MixerExit(self, forward_batch)
return MixerExit(self, forward_batch, stream=stream)
def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit:
def ffn_exit(
self, forward_batch: ForwardBatch, *, stream: ResidualStream
) -> FfnExit:
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
return FfnExit(self, forward_batch)
return FfnExit(self, forward_batch, stream=stream)
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
def _ffn_sum_can_move_to_next_layer(self, steps) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if (
(
self.plan.fusions is not None
and self.plan.fusions.requires_local_reduction
)
or is_enable_moe_cp_allgather()
or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable
):
if is_enable_moe_cp_allgather() or steps.output.group is None:
return False
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
@@ -324,6 +317,7 @@ class OutputBoundary:
def _ffn_sum_moves_to_next_layer(
self,
forward_batch: ForwardBatch,
steps,
*,
mlp_reduce_scatter: bool,
dp_step: Optional[Callable],
@@ -333,8 +327,8 @@ class OutputBoundary:
same all-reduce the FFN itself would have."""
return (
get_parallel().tp_size > 1
and not self.plan.is_last_layer
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and not self.plan.terminal
and self._ffn_sum_can_move_to_next_layer(steps)
and _can_defer_ffn_reduction(forward_batch, self.plan)
and not mlp_reduce_scatter
# Under attention DP the next layer must also run postprocess's
@@ -342,7 +336,7 @@ class OutputBoundary:
and (
not is_dp_attention_enabled()
or (
self._local_token_move_can_go_to_next_layer(forward_batch)
self._local_token_move_can_go_to_next_layer(steps)
and dp_step is None
)
)
@@ -420,16 +414,23 @@ class MixerExit:
"_stream",
)
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
produced = communicator.plan._batch_steps(forward_batch).ffn_output
self._stream = None
def __init__(
self,
boundary: OutputBoundary,
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
):
steps = boundary.plan._batch_steps(forward_batch)
produced = steps.output
self._stream = stream
self._update = produced.update
self._declared_sum = produced.group if produced.always_leaves else None
self._hands_on = (
produced.leaves_for_next_layer
and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
and boundary._ffn_sum_can_move_to_next_layer(steps)
)
self.skips_reduction = produced.always_leaves or self._hands_on
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
@@ -441,21 +442,15 @@ class MixerExit:
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(self, hidden_states: torch.Tensor, residual=None):
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
if self._stream is not None:
residual = self._stream
def finish(self, hidden_states: torch.Tensor):
"""Hand the actual mixer result to the next boundary through the stream."""
if self._hands_on:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
output = residual.leave(
hidden_states, self._update, declared_sum=self._declared_sum
)
return output if self._stream is not None else (output, residual)
return hidden_states
return self._stream.leave(
hidden_states, self._update, declared_sum=self._declared_sum
)
class FfnExit:
@@ -464,8 +459,7 @@ class FfnExit:
``defer_moe_finalize`` on ``get_forward()``."""
__slots__ = (
"communicator",
"forward_batch",
"boundary",
"defer_moe_finalize",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
@@ -476,18 +470,23 @@ class FfnExit:
"_stream",
)
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
self._stream = None
self.communicator = communicator
self.forward_batch = forward_batch
completion = communicator._select_ffn_completion(forward_batch)
def __init__(
self,
boundary: OutputBoundary,
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
):
self._stream = stream
self.boundary = boundary
steps = boundary.plan._batch_steps(forward_batch)
completion = boundary._select_ffn_completion(forward_batch, steps)
self.defer_moe_finalize = completion.defer_moe_finalize
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
steps = communicator.plan._batch_steps(forward_batch)
self._update = steps.ffn_output.update
self._declared_sum = communicator._declared_ffn_sum(
self._update = steps.output.update
self._declared_sum = boundary._declared_ffn_sum(
steps, completion.mlp_reduce_scatter
)
self._scope = get_forward().scoped(
@@ -503,21 +502,14 @@ class FfnExit:
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input, or postprocess."""
if self._stream is not None:
residual = self._stream
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
residual = stream.residual
def finish(self, hidden_states: torch.Tensor):
"""Complete or carry this output, preserving its producer update."""
residual = self._stream.residual
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
# A deferred MoE finalize handoff, consumed by the next attention input.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
else:
hidden_states, residual = self._complete(hidden_states, residual)
result = self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update, self._declared_sum
return self.boundary._leave_ffn_output(
hidden_states, residual, self._stream, self._update, self._declared_sum
)
return result[0] if self._stream is not None else result
+135 -200
View File
@@ -9,21 +9,26 @@ from typing import Mapping, Optional
import msgspec
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.communicator.boundary import (
from sglang.srt.layers.communicator.adapters.overlap import (
resolve_handoff_rows,
tbo_handoff,
)
from sglang.srt.layers.communicator.boundary import _cp_moves
from sglang.srt.layers.communicator.construction import (
BatchVariant,
_bind_stage,
_input_can_be_scattered,
_refuse_uncovered_cp_moe,
_use_ag_after_qlora,
)
from sglang.srt.layers.communicator.contracts import (
EdgeDecl,
HandoffRows,
ProducerReduction,
StageDecl,
StageInput,
StageKind,
StageOutput,
_cp_moves,
)
from sglang.srt.layers.communicator.construction import (
BatchVariant,
StageEdges,
StagePlan,
_input_can_be_scattered,
_refuse_uncovered_cp_moe,
_use_ag_after_qlora,
)
from sglang.srt.layers.communicator.layout import (
Layout,
@@ -35,10 +40,6 @@ from sglang.srt.layers.communicator.layout import (
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import (
@@ -46,15 +47,12 @@ from sglang.srt.layers.communicator.residual.add_norm import (
NORM_QUANT_READ,
NORM_READ,
)
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
def _variants(*, ordinary_only=False):
def _variants():
yield BatchVariant.ORDINARY
if ordinary_only:
return
if _generic_prefill_cp_shards_tokens():
yield BatchVariant.CONTEXT_PARALLEL
if _input_can_be_scattered():
@@ -79,27 +77,31 @@ def _ffn_decl(
*,
sparse,
terminal=False,
next_sparse=False,
output=None,
output_transform=None,
read=NORM_READ,
update=ADD,
local_dense=True,
dense_tp_size=None,
reduction=ProducerReduction.SCOPED,
):
parallel = get_parallel()
reduction = get_exec().comm.boundary_reduction
if reduction not in ("ar", "rs", "rsv", "rs+rsv"):
if dense_tp_size not in (None, 1, parallel.tp_size):
raise ValueError("dense FFN rows support local compute or the full TP group")
strategy = get_exec().comm.boundary_reduction
if strategy not in ("ar", "rs", "rsv", "rs+rsv"):
raise ValueError(
"boundary_reduction must be resolved before model construction"
)
can_move_output = output is None or output.before_reduce_scatter
use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output
use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output
can_move_output = output_transform is None or output_transform.before_reduce_scatter
use_reduce_scatter = strategy in ("rs", "rs+rsv") and can_move_output
use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output
cp_shards = _generic_prefill_cp_shards_tokens()
axes, attention, local, full = _rows(variant)
on_local = (
is_moe_input_scattered_across_dp_ranks()
if sparse
else (local_dense and enable_moe_dense_fully_dp())
else (
enable_moe_dense_fully_dp() if dense_tp_size is None else dense_tp_size == 1
)
)
if parallel.attn_cp_size > 1 and sparse:
_refuse_uncovered_cp_moe(on_local, cp_shards)
@@ -118,7 +120,7 @@ def _ffn_decl(
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=read,
),
StageOutput(local, update=update, transform=output),
StageOutput(local, update=update, transform=output_transform),
),
local,
local,
@@ -136,7 +138,7 @@ def _ffn_decl(
leaves_for_reduce_scatter=use_reduce_scatter
and (scattered_residual or not terminal),
update=update,
transform=output,
transform=output_transform,
),
),
residual,
@@ -156,8 +158,8 @@ def _ffn_decl(
*((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes
)
)
output = (
StageOutput(rows, update=update, transform=output)
produced = (
StageOutput(rows, update=update, transform=output_transform)
if on_local
else StageOutput(
rows,
@@ -166,22 +168,17 @@ def _ffn_decl(
and not terminal
and not update.at_producer
and update.can_defer_across_layers
and output is None,
and output_transform is None
and reduction is ProducerReduction.SCOPED,
leaves_for_reduce_scatter=use_reduce_scatter and may_scatter,
leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave,
update=update,
transform=output,
transform=output_transform,
)
)
gathers_for_tbo = (
enable_moe_dense_fully_dp()
and get_exec().overlap.enable_two_batch_overlap
and not sparse
and bool(next_sparse)
)
returned = local if on_local and not terminal and not gathers_for_tbo else attention
returned = local if on_local and not terminal else attention
return (
StageDecl(StageInput(rows, read=read), output),
StageDecl(StageInput(rows, read=read), produced),
local if on_local else attention,
returned,
)
@@ -201,13 +198,11 @@ class StageDeclaration:
update: StageUpdate
sparse: bool = False
terminal: bool = False
next_sparse: bool = False
output: Optional[OutputTransform] = None
ordinary_only: bool = False
# A mixer with its own exit publishes reduction decisions before compute.
mixer_exit: bool = False
next_kind: Optional[StageKind] = None
return_to_attention: bool = False
output_transform: Optional[OutputTransform] = None
reduction: ProducerReduction = ProducerReduction.SCOPED
gathers_tp_input: bool = False
dense_tp_size: Optional[int] = None
handoff_rows: Optional[HandoffRows] = None
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -245,10 +240,11 @@ def declare_attn(
read=NORM_QUANT_READ,
update=ADD,
terminal=False,
ordinary_only=False,
mixer_exit=False,
next_kind=None,
reduction=ProducerReduction.PARTIAL,
gathers_tp_input=True,
):
if reduction is ProducerReduction.LOCAL_TAIL:
raise ValueError("LOCAL_TAIL is not supported for attention stages")
return StageDeclaration(
StageKind.ATTENTION,
read,
@@ -256,9 +252,8 @@ def declare_attn(
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
ordinary_only=ordinary_only,
mixer_exit=mixer_exit,
next_kind=next_kind,
reduction=reduction,
gathers_tp_input=gathers_tp_input,
)
@@ -270,11 +265,14 @@ def declare_ffn(
read=NORM_READ,
update=ADD,
terminal=False,
output_transform=None,
next_sparse=False,
output=None,
ordinary_only=False,
return_to_attention=False,
dense_tp_size=None,
reduction=ProducerReduction.SCOPED,
handoff_rows=None,
):
if reduction is ProducerReduction.PARTIAL:
raise ValueError("PARTIAL is not supported for ffn stages")
return StageDeclaration(
StageKind.FFN,
read,
@@ -283,43 +281,43 @@ def declare_ffn(
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
next_sparse=next_sparse,
output=output,
ordinary_only=ordinary_only,
return_to_attention=return_to_attention,
output_transform=output_transform,
dense_tp_size=dense_tp_size,
reduction=reduction,
handoff_rows=handoff_rows or tbo_handoff(sparse, next_sparse),
)
def _resolve(stage, variant):
def _resolve(stage, variant, following=None):
axes, attention, local, full = _rows(variant)
if stage.update.at_producer:
if stage.sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
)
if stage.kind is StageKind.FFN:
if stage.update.at_producer:
if stage.sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
)
declaration, residual, returned = _ffn_decl(
variant,
sparse=stage.sparse,
terminal=stage.terminal,
next_sparse=stage.next_sparse,
output=stage.output,
output_transform=stage.output_transform,
read=stage.read,
update=stage.update,
local_dense=not stage.return_to_attention,
dense_tp_size=stage.dense_tp_size,
reduction=stage.reduction,
)
if stage.return_to_attention:
if resolve_handoff_rows(stage.handoff_rows) is HandoffRows.ATTENTION:
returned = attention
return declaration, residual, returned
sp = variant is BatchVariant.SEQUENCE_PARALLEL
scattered = variant is BatchVariant.INPUT_SCATTERED
gathers = (
frozenset({TokenAxis.ATTN_TP_SCATTER})
if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora)
if stage.gathers_tp_input and (sp or scattered or _use_ag_after_qlora)
else frozenset()
)
owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1
@@ -329,10 +327,14 @@ def _resolve(stage, variant):
local if sp else attention,
group=SumGroup.ATTN_TP if owes else None,
always_leaves=owes
and (not stage.mixer_exit or stage.next_kind is StageKind.FFN),
and (
stage.reduction is ProducerReduction.PARTIAL
or (following is not None and following.kind is StageKind.FFN)
),
leaves_for_next_layer=owes
and stage.mixer_exit
and stage.next_kind is StageKind.ATTENTION,
and stage.reduction is ProducerReduction.SCOPED
and following is not None
and following.kind is StageKind.ATTENTION,
update=stage.update,
),
)
@@ -351,9 +353,8 @@ def _connect(producer, consumer, *, residual_from=None):
if producer is None and consumer is None:
raise ValueError("a boundary needs at least one declared side")
before, after = producer, consumer
ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None)
exits, entries = {}, {}
for variant in _variants(ordinary_only=ordinary_only):
for variant in _variants():
_, attention, local, _ = _rows(variant)
if before is None:
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
@@ -367,13 +368,16 @@ def _connect(producer, consumer, *, residual_from=None):
residual = rows
capabilities = (True,)
else:
decl, during, returned = _resolve(before, variant)
decl, during, returned = _resolve(before, variant, following=after)
if during is None:
if residual_from is not None:
if residual_from.consumer != producer:
raise ValueError("residual source must enter the producer")
during = residual_from.entries[variant].residual_to
elif before.mixer_exit:
elif (
before.kind is StageKind.ATTENTION
and before.reduction is ProducerReduction.SCOPED
):
during = attention
else:
raise ValueError(
@@ -382,16 +386,24 @@ def _connect(producer, consumer, *, residual_from=None):
exits[variant] = EdgeDecl(
decl.output, StageInput(returned), during, returned
)
if before.kind is StageKind.ATTENTION and not before.mixer_exit:
if (
before.kind is StageKind.ATTENTION
and before.reduction is ProducerReduction.PARTIAL
):
arrived, residual, capabilities = decl.output, during, ()
else:
owes = (
variant is BatchVariant.INPUT_SCATTERED
and not before.update.at_producer
)
carries = (before.mixer_exit or before.return_to_attention) and (
decl.output.always_leaves or decl.output.leaves_for_next_layer
)
carries = (
(
before.kind is StageKind.ATTENTION
and before.reduction is ProducerReduction.SCOPED
)
or resolve_handoff_rows(before.handoff_rows)
is HandoffRows.ATTENTION
) and (decl.output.always_leaves or decl.output.leaves_for_next_layer)
arrived = StageOutput(
returned,
group=decl.output.group
@@ -413,10 +425,10 @@ def _connect(producer, consumer, *, residual_from=None):
if variant is BatchVariant.INPUT_SCATTERED
and arrived.always_leaves
and arrived.update is None
and not after.mixer_exit
and after.reduction is ProducerReduction.PARTIAL
else residual
)
elif after.return_to_attention:
elif resolve_handoff_rows(after.handoff_rows) is HandoffRows.ATTENTION:
during = (
decl.input.layout
if residual.sharded <= decl.input.layout.sharded
@@ -440,7 +452,7 @@ def _connect(producer, consumer, *, residual_from=None):
if (
before is not None
and before.kind is StageKind.ATTENTION
and not before.mixer_exit
and before.reduction is ProducerReduction.PARTIAL
):
exits[variant] = edge
return StageConnection(producer, consumer, exits, entries)
@@ -453,7 +465,11 @@ def _fork_input(prepared, consumer):
execute the branch's input norm again.
"""
entries = {}
for variant in _variants(ordinary_only=consumer.ordinary_only):
for variant in _variants():
if variant is not BatchVariant.ORDINARY:
raise NotImplementedError(
"prepared branch transport requires ordinary token rows"
)
source = prepared.entries[variant]
declaration, during, _ = _resolve(consumer, variant)
entries[variant] = EdgeDecl(
@@ -476,7 +492,7 @@ def _incoming(stage):
_incoming(previous)
if previous is not None
and previous.kind is StageKind.ATTENTION
and not previous.mixer_exit
and previous.reduction is ProducerReduction.PARTIAL
else None
)
return _connect(
@@ -498,78 +514,12 @@ def _connections(stage, following=None):
return incoming, outgoing
def _bind_stage(declaration, norm, incoming, outgoing, **options):
if incoming.consumer != declaration or outgoing.producer != declaration:
raise ValueError("connections do not match the stage declaration")
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
if declaration.update.at_producer:
if declaration.sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
)
variants = {}
for variant, edge in incoming.entries.items():
if (
declaration.update.at_producer
and TokenAxis.ATTN_CP
in edge.produced.layout.sharded - edge.need.layout.sharded
):
raise NotImplementedError("MHC with a gather over attention CP")
handoff = None
if declaration.kind is StageKind.ATTENTION:
handoff = (
_hand_scattered_input_to_attention
if variant is BatchVariant.INPUT_SCATTERED
and not declaration.update.adds_plainly
else _hand_qkv_hook_its_input
)
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
plan = StagePlan(
declaration.kind,
norm,
variants,
enters_stack=incoming.producer is None,
prepared_input=declaration.prepared_from is not None,
terminal=declaration.terminal,
fixed_output=declaration.kind is StageKind.ATTENTION
and not declaration.mixer_exit,
is_sparse=declaration.sparse,
**options,
)
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
# Layout eligibility comes from the connected consumer, not a mutable
# link to its execution plan. Kernel binding remains consumer-owned.
from sglang.srt.layers.communicator.boundary import make_boundary
plan._fusion_rows = {
v: make_boundary(
edge,
cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None,
force_layernorm_before_gather=options.get(
"force_layernorm_before_dp_gather", False
),
).input_rows
for v, edge in outgoing.entries.items()
}
return StageCommunicator(plan, declaration.kind, norm, declaration=declaration)
def make_attn_stage(
*,
declaration,
norm,
following: Optional[StageDeclaration] = None,
qkv_latent_func=None,
force_layernorm_before_dp_gather=False,
enable_fused_ar_quant=False,
fused_ar_quant_keep_bf16=False,
residual_in_hidden=False,
fusions=None,
):
"""Resolve this attention's boundaries, then bind its norm and hooks.
@@ -587,10 +537,6 @@ def make_attn_stage(
incoming,
outgoing,
qkv_latent_func=qkv_latent_func,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
residual_in_hidden=residual_in_hidden,
fusions=fusions,
)
@@ -600,7 +546,6 @@ def make_ffn_stage(
declaration,
norm,
following: Optional[StageDeclaration] = None,
force_layernorm_before_dp_gather=False,
fusions=None,
):
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
@@ -612,13 +557,13 @@ def make_ffn_stage(
norm,
incoming,
outgoing,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
residual_in_hidden=declaration.update.at_producer,
fusions=fusions,
)
def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
def make_stages(
*stages, previous=None, prepared_from=None, following=None, terminal=False
):
"""Bind a local linear sequence and return its independent boundaries.
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
@@ -653,53 +598,43 @@ def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
previous=previous if index == 0 else declarations[-1],
prepared_from=prepared_from if index == 0 else None,
terminal=terminal and index == len(stages) - 1,
next_kind=stages[index + 1][0].kind
if declaration.mixer_exit and index + 1 < len(stages)
else declaration.next_kind,
)
declarations.append(declaration)
bindings.append((norm, item[2] if len(item) == 3 else {}))
external_consumer = following
boundaries = []
incoming = _incoming(declarations[0])
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
build = (
make_attn_stage
if declaration.kind is StageKind.ATTENTION
else make_ffn_stage
)
boundaries.append(
build(
declaration=declaration,
norm=norm,
following=declarations[index + 1]
if index + 1 < len(declarations)
else None,
**options,
)
following = (
declarations[index + 1]
if index + 1 < len(declarations)
else external_consumer
)
outgoing = _connect(declaration, following, residual_from=incoming)
options = dict(options)
if declaration.kind is StageKind.FFN:
unexpected = options.keys() - {
"fusions",
}
if unexpected:
raise TypeError(f"unsupported FFN options: {sorted(unexpected)}")
boundaries.append(_bind_stage(declaration, norm, incoming, outgoing, **options))
incoming = outgoing
for producer, consumer in zip(boundaries, boundaries[1:]):
if (
producer.kind is not StageKind.ATTENTION
or producer.declaration.mixer_exit
or consumer.kind is not StageKind.FFN
if not (
producer.kind is StageKind.ATTENTION
and producer.declaration.reduction is ProducerReduction.PARTIAL
and consumer.kind is StageKind.FFN
):
continue
for variant, steps in producer.plan._paths.items():
predicate = consumer.plan._paths[variant].ffn.preserves_residual
next_steps = consumer.plan._paths[variant]
predicate = next_steps.entry.preserves_residual
if predicate is not None:
producer.plan._paths[variant] = msgspec.structs.replace(
steps,
attention=msgspec.structs.replace(
steps.attention, capture_preserves_residual=predicate
entry=msgspec.structs.replace(
steps.entry, capture_preserves_residual=predicate
),
)
producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY]
producer.plan._cp_steps = producer.plan._paths.get(
BatchVariant.CONTEXT_PARALLEL
)
producer.plan._input_scattered_steps = producer.plan._paths.get(
BatchVariant.INPUT_SCATTERED
)
producer.plan._sp_steps = producer.plan._paths.get(
BatchVariant.SEQUENCE_PARALLEL
)
return tuple(boundaries)
@@ -19,23 +19,26 @@ from typing import Callable, Optional, Tuple
import torch
from sglang.srt.layers.communicator.boundary import FusedMlpInput
from sglang.srt.layers.communicator.contracts import FusedMlpInput
from sglang.srt.layers.communicator.layout import SumGroup
from sglang.srt.layers.communicator.output import UnreducedOutput
from sglang.srt.layers.communicator.residual.add_norm import (
NORM_QUANT_READ,
Fp8Input,
NormQuantRead,
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import post_experts_reduction_group
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
def attention_fusions(plan) -> Tuple[Callable, ...]:
def attention_fusions(plan, read=NORM_QUANT_READ) -> Tuple[Callable, ...]:
"""The fused kernels that complete what the previous layer left together
with the residual update and the input norm, in the order they are tried.
Each takes (owed, residual, forward_batch, post_residual_addition) and
@@ -45,12 +48,23 @@ def attention_fusions(plan) -> Tuple[Callable, ...]:
given = plan.fusions.attention_input(plan) if plan.fusions else ()
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
return given
plan._attn_input_fuses_quant = (
plan.enable_fused_ar_quant
fuses_quant = (
isinstance(read, NormQuantRead)
and read.fp8_input is not None
and _use_aiter
and not get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false")
and get_exec().comm.enable_aiter_allreduce_fusion
and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group")
)
return (*given, partial(complete_attention_input, plan))
return (
*given,
partial(
complete_attention_input,
plan,
fuses_quant=fuses_quant,
keep_bf16=fuses_quant and read.fp8_input is Fp8Input.TUPLE_AND_BF16,
),
)
def complete_attention_input(
@@ -59,11 +73,14 @@ def complete_attention_input(
residual: torch.Tensor,
forward_batch: ForwardBatch,
post_residual_addition: Optional[torch.Tensor],
*,
fuses_quant: bool,
keep_bf16: bool,
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""Complete the sum the previous layer left, add it to the residual and
apply the input norm in one aiter or flashinfer kernel; None when the
kernel does not take this batch. The result is not quantized for
``quant_format``."""
kernel does not take this batch. The optional FP8 result follows the consumer read declaration;
the separate ``quant_format`` path remains the read adapter's responsibility."""
if (
not isinstance(owed, UnreducedOutput)
# The kernel does not add it.
@@ -78,14 +95,14 @@ def complete_attention_input(
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
):
return None
if plan._attn_input_fuses_quant:
if fuses_quant:
# Falls back to AR+RMSNorm + separate quant internally when the
# fully-fused kernel cannot service the shape.
quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group(
hidden_states,
residual,
use_attn_tp_group=False,
keep_bf16=plan.fused_ar_quant_keep_bf16,
keep_bf16=keep_bf16,
)
if quant_result is not None:
return quant_result
@@ -208,21 +208,18 @@ class CuteDSLFusion:
# Producer capabilities; consumer kernel selection is independent.
self.hands_off_finalize = False
self.terminal_finalize = False
self.requires_local_reduction = False
def install(
self,
service: CuteDSLFusionService,
*,
hands_off_finalize: bool,
output_is_replicated: bool,
terminal_finalize: bool = False,
) -> None:
"""Install the shared workspace and this producer's capabilities."""
self.service = service
self.hands_off_finalize = hands_off_finalize
self.terminal_finalize = terminal_finalize
self.requires_local_reduction = output_is_replicated
def attention_input(self, layer) -> tuple:
return (
@@ -233,7 +230,7 @@ class CuteDSLFusion:
def ffn_input(self, layer) -> tuple:
parallel = get_parallel()
if (
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
TokenAxis.ATTN_TP_SCATTER not in layer.incoming_residual_rows.sharded
# The workspace sums over TP, which is then the attention-TP group.
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(layer.norm) is not None
@@ -336,11 +333,7 @@ class CuteDSLFusion:
"""Whether this producer can emit a handoff with an unfused fallback."""
return (
self.hands_off_finalize
and (not layer.is_last_layer or self.terminal_finalize)
and (
not self.requires_local_reduction
or (layer.is_last_layer and self.terminal_finalize)
)
and (not layer.terminal or self.terminal_finalize)
and self._should_use_finalize(
layer, forward_batch, int(forward_batch.input_ids.shape[0])
)
@@ -377,7 +370,6 @@ def install_cutedsl_fusion(
top_k: int,
rms_epsilon: float,
can_defer_finalize: _LayerPredicate,
requires_local_reduction: _LayerPredicate | None = None,
label: str,
terminal_finalize: bool = False,
) -> CuteDSLFusionService | None:
@@ -424,8 +416,6 @@ def install_cutedsl_fusion(
service,
hands_off_finalize=hands_off_finalize,
terminal_finalize=terminal_finalize,
output_is_replicated=requires_local_reduction is not None
and bool(requires_local_reduction(layer)),
)
logger.info(
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
File diff suppressed because it is too large Load Diff
@@ -142,7 +142,9 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
return parallel.attn_tp_group
if group is SumGroup.TP:
return parallel.tp_group
return post_experts_reduction_group()
if group is SumGroup.MOE_OUTPUT:
return post_experts_reduction_group()
raise ValueError(f"unsupported output sum group: {group!r}")
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
@@ -1,160 +0,0 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A stage's model-facing input and output boundary."""
from typing import TYPE_CHECKING, Optional
from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.communicator.residual.batch import current
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
if TYPE_CHECKING:
from sglang.srt.layers.communicator.layer import LayerCommunicator
class StageCommunicator:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
self._layer = layer
self._name = name
self.norm = getattr(layer, norm_name)
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
) -> StageEntry:
if steps is None:
steps = self._layer._batch_steps(forward_batch)
entry = getattr(steps, self._name)
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self._name} stage"
)
return entry
def _prepare(
self,
hidden_states,
residual,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
stream = current(forward_batch)
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
if (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
):
hidden_states = stream.leave(
hidden_states, ADD, declared_sum=entry.input_sum
)
if stream.pending is not None:
call["update"] = stream.pending.update
call["pending"] = stream.pending
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, **call
)
if entry.input_move is not None:
hidden_states = entry.input_move(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
if entry.handoff is not None:
hidden_states = entry.handoff(
hidden_states, forward_batch, self._layer.qkv_latent_func
)
stream.write(residual)
return hidden_states, stream
@property
def input_rows(self):
return self._layer.input_rows
@property
def input_on_attention_tp_slices(self):
return self._layer.input_on_attention_tp_slices
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
stream = current(forward_batch)
if self._name == "attention":
hidden_states, stream = (
self._layer.prepare_attn_and_capture_last_layer_outputs(
hidden_states, stream, forward_batch, **call
)
)
else:
steps = self._layer._batch_steps(forward_batch)
self._layer.publish_mlp_lora_layout(steps)
if cache is not None:
call["cache"] = cache
hidden_states, stream = self._prepare(
hidden_states, stream, forward_batch, steps, **call
)
forward_batch.residual_stream = stream
return hidden_states
def finish(self, hidden_states, forward_batch):
"""Hand this attention's actual contribution to the following stage."""
if self._name != "attention" or self._layer.stage_edges is not None:
raise RuntimeError(
"use the stage exit scope for an FFN or single-stage mixer"
)
return current(forward_batch).leave(
hidden_states,
self._layer._residual.attention_update,
declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum,
)
def exit(self, forward_batch):
result = (
self._layer.ffn_exit(forward_batch)
if self._name == "ffn"
else self._layer.mixer_exit(forward_batch)
)
result._stream = current(forward_batch)
return result
def from_pp(self, tensors, forward_batch, **kwargs):
hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs)
forward_batch.residual_stream = stream
return hidden_states
def snapshot(self, hidden_states, forward_batch, *, at_input=False):
return self._layer.snapshot(
hidden_states, current(forward_batch), at_input=at_input
)
def capture_output(self, hidden_states, forward_batch, **kwargs):
return self._layer.capture_output(
hidden_states, current(forward_batch), **kwargs
)
def postprocess(self, hidden_states, forward_batch):
hidden_states, stream = self._layer.postprocess_layer(
hidden_states, current(forward_batch), forward_batch
)
forward_batch.residual_stream = stream
return hidden_states
+153 -589
View File
@@ -22,36 +22,25 @@ from sglang.srt.distributed import (
GroupCoordinator,
attention_tensor_model_parallel_all_reduce,
attention_tensor_model_parallel_quant_all_reduce,
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
Layout,
SumGroup,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.communicator.residual import StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
@@ -63,21 +52,11 @@ from sglang.srt.layers.dp_attention import (
get_global_dp_buffer,
get_local_dp_buffer,
get_moe_cp_rank,
get_moe_cp_size,
is_allocation_symmetric,
moe_cp_all_gather_into_tensor,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
is_npu,
)
_is_npu = is_npu()
if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
def tp_reduce_scatter(
@@ -110,43 +89,34 @@ def tp_slice(hidden_states, residual):
return hidden_states, residual
class CommunicateSimpleFn:
@staticmethod
def _trivial(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
return hidden_states
@staticmethod
def _scattered_to_tp_attn_full(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
parallel = get_parallel()
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
for local_hidden_states in hidden_states:
with use_symmetric_memory(
parallel.tp_group,
disabled=not is_allocation_symmetric(),
):
output = torch.empty(
(
local_hidden_states.shape[0] * parallel.attn_tp_size,
*local_hidden_states.shape[1:],
),
dtype=local_hidden_states.dtype,
device=local_hidden_states.device,
)
attn_tp_all_gather_into_tensor(
output,
local_hidden_states,
def gather_attention_tp(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
parallel = get_parallel()
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
for local_hidden_states in hidden_states:
with use_symmetric_memory(
parallel.tp_group,
disabled=not is_allocation_symmetric(),
):
output = torch.empty(
(
local_hidden_states.shape[0] * parallel.attn_tp_size,
*local_hidden_states.shape[1:],
),
dtype=local_hidden_states.dtype,
device=local_hidden_states.device,
)
gathered_hidden_states.append(output)
return tuple(gathered_hidden_states)
attn_tp_all_gather_into_tensor(
output,
local_hidden_states,
)
gathered_hidden_states.append(output)
return tuple(gathered_hidden_states)
return _redistribute_from_attn_tp_shards(hidden_states)
return _redistribute_from_attn_tp_shards(hidden_states)
def _reduce_and_redistribute_output_to_attn_tp_shards(
@@ -244,145 +214,6 @@ def move_rows(
return hidden_states
def _mlp_input_without_dp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
group: SumGroup = SumGroup.ATTN_TP,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Complete the sum the input owes over ``group`` on the rows it is on,
unless one of ``fusions`` does it with the residual add and the norm, then
write it into the residual and read the input."""
if gathers_residual:
residual = update.residual_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
if group is SumGroup.ATTN_TP:
# MHC sums its streams in full precision.
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=update.adds_plainly
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if _is_npu and cache is not None:
_ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_dp_replicate(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: complete the attention-TP sum if it is owed, write the
output into the residual and read the FFN input locally, then gather. With
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
beside the others' in the group's slot."""
if gathers_residual:
residual = update.residual_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
else:
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
return (
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
residual,
)
def _mlp_input_dp_partial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: one rank adds the residual, the gather sums it, then read
the input from the sum. With ``places_cp_shards`` each CP rank puts its
shard of the DP group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if get_parallel().attn_tp_rank == 0:
hidden_states += residual
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
hidden_states = _reduce_and_redistribute_output_to_dp(
hidden_states, forward_batch, cp_shard_counts
)
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
hidden_states, _ = read.read(hidden_states, layernorm)
return hidden_states, residual
def _hand_qkv_hook_its_input(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Give the attention's QKV hook the attention input."""
if qkv_latent_func is not None:
get_attn_tp_context().set_attn_inputs(
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
)
return hidden_states
def _hand_scattered_input_to_attention(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Input-scattered attention takes each rank's slice, and its QKV hook
gathers the rows after the projection. DSA and attention without a hook
consume full hidden states, so those are gathered here, and the hook is
told they are."""
ctx = get_attn_tp_context()
if ctx.is_dsa or qkv_latent_func is None:
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
if qkv_latent_func is not None:
ctx.set_attn_inputs(
AttentionInputs(
hidden_states,
forward_batch,
qkv_latent_func,
is_pre_gathered=ctx.is_dsa,
)
)
return hidden_states
def _tp_all_gather_scattered_rows(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
@@ -393,267 +224,6 @@ def _tp_all_gather_scattered_rows(
return output
def _dispatch_consumer(
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
):
if residual is None:
# Enter/read-only paths do not execute an update.
prepare = paths[False]
else:
try:
prepare = paths[update.adds_plainly]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
def _consumer_step(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
*,
step: Callable,
adds_plainly: bool,
carried_fusions: Tuple[Callable, ...],
expected_sum: Optional[SumGroup] = None,
completed_step: Optional[Callable] = None,
pending=None,
written_step: Optional[Callable] = None,
written: bool = False,
update: StageUpdate = ADD,
**call,
):
"""A boundary's half into a stage: complete what the value carries, a sum
or a handoff its producer left for this batch, then run ``step`` on the
complete value. Under attention DP the reduction back to this rank's tokens
comes first; otherwise one of ``carried_fusions`` may complete it with the
residual add and the read, and an empty batch has nothing to sum. An input
whose declared sum is already completed uses ``completed_step`` instead;
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
``quant_format`` and ``post_residual_addition``)."""
if residual is not None and update.adds_plainly != adds_plainly:
raise RuntimeError("producer update does not match the boundary's capability")
if pending is not None:
if isinstance(pending.owed, DeclaredSum):
if pending.owed.group is not expected_sum:
raise RuntimeError("producer sum does not match the input declaration")
elif expected_sum is not None:
if pending.owed is not None:
raise RuntimeError("a declared input sum carried another completion")
step = completed_step
if written and written_step is not None:
step = written_step
if not isinstance(hidden_states, torch.Tensor):
owed = hidden_states
if residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
if expected_sum is not None:
raise RuntimeError(
f"an input that owes its sum by construction arrived as "
f"{type(owed).__name__}"
)
if (
isinstance(owed, UnreducedOutput)
and owed.reduce_and_redistribute is not None
):
hidden_states = reduce_output(owed)
else:
for fused in carried_fusions:
result = fused(
owed, residual, forward_batch, call.get("post_residual_addition")
)
if result is not None:
return result
if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0:
hidden_states = reduce_output(owed)
else:
hidden_states = owed.partial
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
def _read_input(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
*,
cache=None,
layer_input: Optional[Callable],
enters_stack: bool,
read: StageRead,
update: StageUpdate = ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
"""Complete what the input owes by construction (``layer_input``), then
write the previous stage's output into the residual and read the stage's
input with ``norm``. The layer stack's first stage (``enters_stack``)
starts its residual from its input."""
enters = residual is None and enters_stack
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual)
if enters:
hidden_states, residual = read.enter(hidden_states), None
if residual is None:
# The previous layer already wrote its output into the residual.
return read.read(hidden_states, norm, quant_format)
return read.update_and_read(
update,
hidden_states,
residual,
norm,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
def _mlp_input_scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
if scatters_residual:
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_slice(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
if scatters_residual and residual is not None:
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
reduces: bool = True,
read: StageRead,
update: StageUpdate,
):
"""The residual stays on each rank's slice while the FFN takes the full
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
if reduces:
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
else:
shard = shard.clone()
shard, residual = read.update_and_read(update, shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
def _mlp_input_residual_into_sum(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, read
)
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
read: StageRead = NORM_READ,
):
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
return read.read(residual, layernorm)
def _mlp_input_gather_attention_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
return dsa_cp_gather_hidden_states(hidden_states), residual
def _mlp_input_gather_moe_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
TP_ATTN_FULL."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None and hidden_states.shape[0] > 0:
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, rows, get_moe_cp_size()
)
return hidden_states, residual
def _dp_scatter_group() -> GroupCoordinator:
parallel = get_parallel()
if parallel.tp_size == parallel.attn_dp_size:
@@ -720,140 +290,134 @@ def _all_reduce_then_to_local_tokens(
)
class CommunicateSummableTensorPairFn:
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
def identity_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
return hidden_states, residual
@staticmethod
def _trivial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
def update_and_gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
update: StageUpdate = ADD,
**kwargs,
):
hidden_states = update.update(hidden_states, residual)
return _redistribute_from_attn_tp_shards(hidden_states), None
def output_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
*,
sums: bool,
gathers_back: bool,
update: StageUpdate,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
parallel = get_parallel()
if sums:
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
else:
hidden_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
if not gathers_back:
return hidden_states, residual
local_states = update.update(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
)
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
def _gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
update: StageUpdate = ADD,
**kwargs,
):
hidden_states = update.update(hidden_states, residual)
return _redistribute_from_attn_tp_shards(hidden_states), None
@staticmethod
def _onto_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
*,
sums: bool,
gathers_back: bool,
update: StageUpdate,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
parallel = get_parallel()
if sums:
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
else:
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
parallel.tp_rank
]
if not gathers_back:
return hidden_states, residual
local_states = update.update(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
def scatter_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
assert residual is None, "not yet handled residual!=None"
return _redistribute_to_attn_tp_shards(hidden_states), None
def take_back_attention_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
equal shards over the attention-CP group."""
parallel = get_parallel()
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
return shard, residual
def reduce_scatter_over_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
keep this rank's shard."""
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
def take_back_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
slot, at the shard's padded length with the padding zeroed."""
held = forward_batch.attn_cp_metadata.per_rank_actual_token
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
: held[get_parallel().attn_cp_rank]
]
dp_scatter(
local_hidden_states,
hidden_states,
forward_batch,
_cp_shard_token_rows(forward_batch),
)
return local_hidden_states, residual
def scatter_moe_cp_output(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
CP-local portion.
If DP>1, further scatter back to the local DP slice.
"""
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
# — CP extend always has non-zero tokens per rank, and decode skips this path.
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
if get_parallel().attn_dp_size > 1:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
def _scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
assert residual is None, "not yet handled residual!=None"
return _redistribute_to_attn_tp_shards(hidden_states), None
@staticmethod
def _take_back_attention_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
equal shards over the attention-CP group."""
parallel = get_parallel()
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
return shard, residual
@staticmethod
def _reduce_scatter_over_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
keep this rank's shard."""
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
@staticmethod
def _take_back_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
slot, at the shard's padded length with the padding zeroed."""
held = forward_batch.attn_cp_metadata.per_rank_actual_token
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
: held[get_parallel().attn_cp_rank]
]
dp_scatter(
local_hidden_states,
hidden_states,
forward_batch,
_cp_shard_token_rows(forward_batch),
)
return local_hidden_states, residual
@staticmethod
def _scatter_hidden_states_moe(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
CP-local portion.
If DP>1, further scatter back to the local DP slice.
"""
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
# — CP extend always has non-zero tokens per rank, and decode skips this path.
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
if get_parallel().attn_dp_size > 1:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
return hidden_states, residual
return hidden_states, residual
@@ -0,0 +1,475 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Complete pending output, update the residual, and read the consumer input."""
from typing import Callable, Optional, Tuple, Union
import torch
from sglang.srt.distributed import (
attention_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_gather_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
SumGroup,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
dp_scatter,
get_moe_cp_size,
is_allocation_symmetric,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import (
is_npu,
)
_is_npu = is_npu()
if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
from sglang.srt.layers.communicator.ops import (
_mlp_input_reduce_output,
_redistribute_input_to_dp,
_redistribute_input_to_moe_cp,
_reduce_and_redistribute_output_to_attn_tp_shards,
_reduce_and_redistribute_output_to_dp,
_tp_all_gather_scattered_rows,
)
def _mlp_input_without_dp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
group: SumGroup = SumGroup.ATTN_TP,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Complete the sum the input owes over ``group`` on the rows it is on,
unless one of ``fusions`` does it with the residual add and the norm, then
write it into the residual and read the input."""
if gathers_residual:
residual = update.residual_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
if group is SumGroup.ATTN_TP:
# MHC sums its streams in full precision.
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=update.adds_plainly
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if _is_npu and cache is not None:
_ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_dp_replicate(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: complete the attention-TP sum if it is owed, write the
output into the residual and read the FFN input locally, then gather. With
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
beside the others' in the group's slot."""
if gathers_residual:
residual = update.residual_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
else:
hidden_states, residual = read.update_and_read(
update, hidden_states, residual, layernorm
)
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
return (
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
residual,
)
def _mlp_input_dp_partial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
"""Attention DP: one rank adds the residual, the gather sums it, then read
the input from the sum. With ``places_cp_shards`` each CP rank puts its
shard of the DP group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if get_parallel().attn_tp_rank == 0:
hidden_states += residual
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
hidden_states = _reduce_and_redistribute_output_to_dp(
hidden_states, forward_batch, cp_shard_counts
)
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
hidden_states, _ = read.read(hidden_states, layernorm)
return hidden_states, residual
def _hand_qkv_hook_its_input(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Give the attention's QKV hook the attention input."""
if qkv_latent_func is not None:
get_attn_tp_context().set_attn_inputs(
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
)
return hidden_states
def _hand_scattered_input_to_attention(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Input-scattered attention takes each rank's slice, and its QKV hook
gathers the rows after the projection. DSA and attention without a hook
consume full hidden states, so those are gathered here, and the hook is
told they are."""
ctx = get_attn_tp_context()
if ctx.is_dsa or qkv_latent_func is None:
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
if qkv_latent_func is not None:
ctx.set_attn_inputs(
AttentionInputs(
hidden_states,
forward_batch,
qkv_latent_func,
is_pre_gathered=ctx.is_dsa,
)
)
return hidden_states
def _dispatch_consumer(
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
):
if residual is None:
# A missing residual cannot join a partial sum. The non-plain path
# performs enter/read without that optimization.
prepare = paths[False]
else:
try:
prepare = paths[update.adds_plainly]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
def _consumer_step(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
*,
step: Callable,
adds_plainly: bool,
carried_fusions: Tuple[Callable, ...],
expected_sum: Optional[SumGroup] = None,
completed_step: Optional[Callable] = None,
pending=None,
written_step: Optional[Callable] = None,
written: bool = False,
update: StageUpdate = ADD,
**call,
):
"""A boundary's half into a stage: complete what the value carries, a sum
or a handoff its producer left for this batch, then run ``step`` on the
complete value. Under attention DP the reduction back to this rank's tokens
comes first; otherwise one of ``carried_fusions`` may complete it with the
residual add and the read, and an empty batch has nothing to sum. An input
whose declared sum is already completed uses ``completed_step`` instead;
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
``quant_format`` and ``post_residual_addition``)."""
if residual is not None and update.adds_plainly != adds_plainly:
raise RuntimeError("producer update does not match the boundary's capability")
if pending is not None:
if isinstance(pending.owed, DeclaredSum):
if pending.owed.group is not expected_sum:
raise RuntimeError("producer sum does not match the input declaration")
elif expected_sum is not None:
if pending.owed is not None:
raise RuntimeError("a declared input sum carried another completion")
step = completed_step
if written and written_step is not None:
step = written_step
if not isinstance(hidden_states, torch.Tensor):
owed = hidden_states
if residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
if expected_sum is not None:
raise RuntimeError(
f"an input that owes its sum by construction arrived as "
f"{type(owed).__name__}"
)
if (
isinstance(owed, UnreducedOutput)
and owed.reduce_and_redistribute is not None
):
hidden_states = reduce_output(owed)
else:
for fused in carried_fusions:
result = fused(
owed, residual, forward_batch, call.get("post_residual_addition")
)
if result is not None:
return result
if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0:
hidden_states = reduce_output(owed)
else:
hidden_states = owed.partial
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
def _read_input(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
*,
cache=None,
layer_input: Optional[Callable],
enters_stack: bool,
read: StageRead,
update: StageUpdate = ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
"""Complete what the input owes by construction (``layer_input``), then
write the previous stage's output into the residual and read the stage's
input with ``norm``. The layer stack's first stage (``enters_stack``)
starts its residual from its input."""
enters = residual is None and enters_stack
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual)
if enters:
hidden_states, residual = read.enter(hidden_states), None
if residual is None:
# The previous layer already wrote its output into the residual.
return read.read(hidden_states, norm, quant_format)
return read.update_and_read(
update,
hidden_states,
residual,
norm,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
def _mlp_input_scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
if scatters_residual:
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_slice(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
if scatters_residual and residual is not None:
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
reduces: bool = True,
read: StageRead,
update: StageUpdate,
):
"""The residual stays on each rank's slice while the FFN takes the full
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
if reduces:
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
else:
shard = shard.clone()
shard, residual = read.update_and_read(update, shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
def _mlp_input_residual_into_sum(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, read
)
def _mlp_input_gather_attention_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
return dsa_cp_gather_hidden_states(hidden_states), residual
def _mlp_input_gather_moe_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
TP_ATTN_FULL."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None and hidden_states.shape[0] > 0:
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, rows, get_moe_cp_size()
)
return hidden_states, residual
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
read: StageRead = NORM_READ,
):
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
return read.read(residual, layernorm)
@@ -53,6 +53,8 @@ class StageRead(Protocol):
# and the residual is left as it is: what a fused add + norm kernel computes,
# and what may run on the rows a sum completes onto.
norms_plainly: bool
# Preserve the read on the source rows before a DP gather.
before_gather: bool
def enter(self, hidden_states) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
@@ -59,6 +59,8 @@ def fold(hidden_states, residual):
The following stage receives it with no outstanding residual addition."""
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
if stream.pending is not None and not stream.pending.update.adds_plainly:
raise NotImplementedError("fold requires a plain residual update")
hidden_states, residual = stream.finish(hidden_states)
hidden_states = reduce_output(hidden_states)
if residual is not None:
@@ -70,10 +72,9 @@ def fold(hidden_states, residual):
)
def written(hidden_states, forward_batch):
def written(hidden_states):
"""Re-enter after a computation that already updated the full residual."""
forward_batch.residual_stream = ResidualStream(hidden_states)
return hidden_states, forward_batch.residual_stream
return hidden_states, ResidualStream(hidden_states)
def finish_layer_stack(
@@ -90,16 +91,11 @@ def finish_layer_stack(
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
if isinstance(residual, ResidualStream):
stream = forward_batch.residual_stream
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
output = stream.finish(
return residual.finish(
hidden_states,
takes_handoff=final_norm_takes_handoff,
preserve_declared=preserve_declared,
)
forward_batch.residual_stream = None
return output
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
return hidden_states, residual
return reduce_output(hidden_states), residual
@@ -149,7 +145,9 @@ def snapshot(
"""Copy a complete output with its plain residual. A statically declared
sum is reduced on a copy; the main output and residual remain unchanged."""
if isinstance(residual, ResidualStream):
return residual.snapshot(hidden_states, group=group)
if group is not None:
raise ValueError("a residual stream carries its own reduction")
return residual.snapshot(hidden_states)
if group is not None:
hidden_states = group.all_reduce(hidden_states.clone())
return hidden_states.clone() if residual is None else hidden_states + residual
@@ -14,6 +14,8 @@
"""The plain residual: add and RMSNorm, and the fused all-reduce + add + norm
kernels."""
from dataclasses import dataclass
from enum import Enum, auto
from typing import Optional
import torch
@@ -280,6 +282,14 @@ class Add:
return _redistribute_from_attn_tp_shards(residual)
class Fp8Input(Enum):
"""Additional input forms a projection can consume after a fused norm."""
TUPLE = auto()
TUPLE_AND_BF16 = auto()
@dataclass(frozen=True)
class NormQuantRead:
"""The input is the residual's norm, fused with the quantization the
consumer asks for (``quant_format``) and a post-residual addition; a plain
@@ -287,6 +297,8 @@ class NormQuantRead:
the norm."""
norms_plainly = True
before_gather: bool = False
fp8_input: Optional[Fp8Input] = None
def enter(self, hidden_states):
return hidden_states
@@ -316,11 +328,13 @@ class NormQuantRead:
)
@dataclass(frozen=True)
class NormRead:
"""The input is the residual's norm; a plain add of the previous output
runs in the same kernel. An empty batch skips the norm."""
norms_plainly = True
before_gather: bool = False
def enter(self, hidden_states):
return hidden_states
@@ -19,9 +19,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
def start(forward_batch):
stream = ResidualStream()
forward_batch.residual_stream = stream
return stream
forward_batch.residual_stream = ResidualStream()
def current(forward_batch):
@@ -31,7 +29,8 @@ def current(forward_batch):
return stream
def finish(hidden_states, forward_batch):
def complete_output(hidden_states, forward_batch):
"""Complete the contribution while retaining its pending residual update."""
return current(forward_batch).complete(hidden_states)
@@ -123,6 +123,7 @@ class _AttentionRead:
layer's output: an MHC layer takes its input written back."""
norms_plainly = False
before_gather = False
def __init__(self, state: MHCState):
self.state = state
@@ -165,6 +166,7 @@ class _FfnRead:
in hc_ffn_post_pre when it takes the batch."""
norms_plainly = False
before_gather = False
def __init__(self, state: MHCState):
self.state = state
@@ -158,22 +158,23 @@ class ResidualStream:
self.check(hidden)
return self.pending.complete() if self.pending is not None else hidden
def snapshot(self, hidden, *, group=None):
def snapshot(self, hidden):
self.check(hidden)
pending = self.pending
if pending is None:
return hidden.clone()
if not pending.update.adds_plainly:
raise NotImplementedError("snapshot requires a plain residual update")
if pending.owed is None:
value = pending.value
elif isinstance(pending.owed, (CarriedSum, DeclaredSum)):
value = pending.owed.complete(pending.value.clone())
else:
raise NotImplementedError("a finalize handoff requires main-output capture")
if group is not None:
value = group.all_reduce(value.clone())
return value.clone() if self.residual is None else value + self.residual
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
"""Export the contribution/residual pair; keep the stream available to readers."""
self.check(hidden)
if self.pending is None:
return hidden, None
+39 -55
View File
@@ -24,12 +24,13 @@ from sglang.srt.layers.communicator.adapters.lora import (
publish_attention,
publish_ffn,
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
from sglang.srt.layers.communicator.contracts import (
BatchVariant,
StageEntry,
StageKind,
StageSteps,
)
from sglang.srt.layers.communicator.ops import CommunicateSimpleFn
from sglang.srt.layers.communicator.ops import gather_attention_tp
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
@@ -51,58 +52,40 @@ class StageCommunicator:
def __init__(
self,
plan: "StagePlan",
kind: StageKind,
norm,
*,
declaration: Optional["StageDeclaration"] = None,
declaration: "StageDeclaration",
):
self.plan = plan
self.kind = kind
self.norm = norm
self._declaration = declaration
self.declaration = declaration
@property
def declaration(self) -> "StageDeclaration":
"""Immutable source contract for constructing another local sequence."""
if self._declaration is None:
raise RuntimeError("this boundary was constructed from raw layout edges")
return self._declaration
def kind(self):
return self.declaration.kind
@property
def norm(self):
return self.plan.norm
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
self, forward_batch: ForwardBatch, steps: Optional[StageSteps] = None
) -> StageEntry:
if steps is None:
steps = self.plan._batch_steps(forward_batch)
entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self.kind.name.lower()} stage"
)
return entry
return steps.entry
def _prepare(
self,
hidden_states,
residual,
stream: ResidualStream,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
steps: Optional[StageSteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
stream = (
residual
if isinstance(residual, ResidualStream)
else ResidualStream(residual)
)
if not isinstance(residual, ResidualStream):
if residual is not None or entry.input_sum is not None:
hidden_states = stream.leave(
hidden_states, call.get("update", ADD), declared_sum=entry.input_sum
)
elif (
if (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
@@ -136,8 +119,8 @@ class StageCommunicator:
return self.plan.fusions
@property
def input_rows(self):
return self.plan.input_rows
def incoming_residual_rows(self):
return self.plan.incoming_residual_rows
@property
def input_on_attention_tp_slices(self):
@@ -158,11 +141,11 @@ class StageCommunicator:
def finish(self, hidden_states, forward_batch):
"""Hand this attention's actual contribution to the following stage."""
if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output:
if self.kind is not StageKind.ATTENTION or not self.plan.direct_handoff:
raise RuntimeError(
"use the stage exit scope for an FFN or single-stage mixer"
)
produced = self.plan.produced(self.kind, forward_batch)
produced = self.plan.produced(forward_batch)
return current(forward_batch).leave(
hidden_states,
produced.update,
@@ -171,24 +154,24 @@ class StageCommunicator:
def exit(self, forward_batch):
result = (
self.plan.output.ffn_exit(forward_batch)
self.plan.output.ffn_exit(forward_batch, stream=current(forward_batch))
if self.kind is StageKind.FFN
else self.plan.output.mixer_exit(forward_batch)
else self.plan.output.mixer_exit(
forward_batch, stream=current(forward_batch)
)
)
result._stream = current(forward_batch)
return result
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
hidden_states, residual = from_pp(
tensors,
residual_in_hidden=self.plan.residual_in_hidden,
residual_in_hidden=(
self.declaration.previous is not None
and self.declaration.previous.update.at_producer
),
allow_missing_residual=allow_missing_residual,
)
declared_sum = None
if forward_batch is not None:
steps = self.plan._batch_steps(forward_batch)
entry = steps.attention or steps.ffn
declared_sum = entry.input_sum
declared_sum = self.entry(forward_batch).input_sum
hidden_states, forward_batch.residual_stream = ResidualStream.arrive(
hidden_states, residual, ADD, declared_sum=declared_sum
)
@@ -208,11 +191,9 @@ class StageCommunicator:
return hidden_states, stream.snapshot(hidden_states)
def postprocess(self, hidden_states, forward_batch):
hidden_states, stream = self.plan.output.postprocess_layer(
return self.plan.output.postprocess_layer(
hidden_states, current(forward_batch), forward_batch
)
forward_batch.residual_stream = stream
return hidden_states
def branch_input(self, source, hidden_states, forward_batch):
hidden_states, stream = branch.branch_input(
@@ -240,11 +221,14 @@ class StageCommunicator:
forward_batch.residual_stream = stream
return hidden_states
def _prepare_input(self, hidden_states, residual, forward_batch, **call):
def _prepare_input(self, hidden_states, stream, forward_batch, **call):
plan = self.plan
if self.kind is StageKind.ATTENTION:
publish_attention(plan._publish_lora_layout)
if plan._sp_steps is not None and plan.is_first_layer:
if (
plan._paths.get(BatchVariant.SEQUENCE_PARALLEL) is not None
and plan.enters_stack
):
get_forward().set(
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
)
@@ -252,7 +236,7 @@ class StageCommunicator:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
else:
publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows)
return self._prepare(hidden_states, residual, forward_batch, **call)
return self._prepare(hidden_states, stream, forward_batch, **call)
def _prepare_attention(
self,
@@ -315,11 +299,11 @@ class StageCommunicator:
forward_batch=forward_batch,
)
)
# Input gathers allocate fresh DP buffers.
# Input gathers allocate fresh DP buffers (see gather_attention_tp).
# Without a gather this is the mutable residual, so retention copies.
captured_last_layer_outputs.capture(
gathered_last_layer_output,
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
owned=move is gather_attention_tp
or (
move is None
and entry.capture_preserves_residual is not None
+1 -1
View File
@@ -844,7 +844,7 @@ class BailingMoEModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
@@ -57,7 +57,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP, _is_hip
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import (
get_device,
@@ -1007,7 +1007,7 @@ class BailingMoELinearModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
@@ -181,7 +181,7 @@ class BailingMoEModelNextN(nn.Module):
else:
hidden_states = self.decoder(positions, hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.final_layernorm
+1 -1
View File
@@ -1395,7 +1395,7 @@ class BailingMoELinearModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states,
+1 -1
View File
@@ -226,7 +226,7 @@ class DeepseekModelNextN(nn.Module):
zero_allocator,
prev_topk_indices=index_topk_share.topk_indices,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
+2 -8
View File
@@ -2655,7 +2655,7 @@ class DeepseekV2DecoderLayer(nn.Module):
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=self._stage_next_sparse,
output=output,
output_transform=output,
),
post_attention_layernorm,
{"fusions": fusions},
@@ -2998,12 +2998,6 @@ class DeepseekV2Model(nn.Module):
and layer.mlp.num_fused_shared_experts == 0
and not layer.mlp._shared_expert_tp1
),
# A TP1-replicated output must not ride into the next layer's reduction.
requires_local_reduction=lambda layer: (
layer.mlp._shared_expert_tp1
if isinstance(layer.mlp, DeepseekV2MoE)
else layer.mlp.tp_size == 1
),
label="DeepSeek-V3/GLM",
)
@@ -3160,7 +3154,7 @@ class DeepseekV2Model(nn.Module):
proxy_tensors.tensors["topk_indices"] = topk_indices
return proxy_tensors
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
@@ -1795,7 +1795,7 @@ class Dots3Model(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
@@ -119,7 +119,7 @@ class Dot3NoteModelNextN(nn.Module):
hidden_states = head.decoder(
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
+2 -2
View File
@@ -226,7 +226,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
(declare_attn(), self.input_layernorm),
(
declare_ffn(
output=OutputTransform(
output_transform=OutputTransform(
self.feed_forward.scale_output, before_reduce_scatter=True
),
sparse=self.is_layer_sparse,
@@ -439,7 +439,7 @@ class FalconH1Model(nn.Module):
forward_batch=forward_batch,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
+1 -1
View File
@@ -583,7 +583,7 @@ class GigaChat35Model(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -115,7 +115,7 @@ class GigaChat35ModelNextN(nn.Module):
zero_allocator=zero_allocator,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
+1 -1
View File
@@ -1121,7 +1121,7 @@ class Glm4MoeModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
+1 -1
View File
@@ -826,7 +826,7 @@ class Glm4MoeLiteModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
@@ -121,7 +121,7 @@ class Glm4MoeLiteModelNextN(nn.Module):
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
+1 -1
View File
@@ -105,7 +105,7 @@ class Glm4MoeModelNextN(nn.Module):
with get_global_expert_distribution_recorder().disable_this_region():
hidden_states = self.decoder(positions, hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.shared_head.norm
+1 -2
View File
@@ -801,7 +801,6 @@ class Glm5NextDecoderLayer(nn.Module):
),
self.input_layernorm,
{
"residual_in_hidden": residual.ffn_update.at_producer,
"qkv_latent_func": self.self_attn.prepare_qkv_latent
if not self.is_linear_attn
else None,
@@ -1192,7 +1191,7 @@ class Glm5NextModel(nn.Module):
return PPProxyTensors({"hidden_states": hidden_states})
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
+1 -1
View File
@@ -728,7 +728,7 @@ class GptOssModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states,
+14 -19
View File
@@ -20,6 +20,7 @@ from sglang.srt.layers.communicator import (
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import (
@@ -49,7 +50,6 @@ from sglang.srt.models.qwen3_5 import (
Qwen3_5ForCausalLM,
Qwen3_5ForConditionalGeneration,
Qwen3_5GatedDeltaNet,
_enable_qwen35_fused_ar_quant,
_linear_accepts_fp8_tuple,
)
from sglang.srt.runtime_context import get_parallel, get_stream
@@ -488,18 +488,15 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
self.post_attention_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant()
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
declare_attn(
read=NormQuantRead(
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
},
),
(
declare_ffn(
@@ -623,17 +620,15 @@ class InternS2MobiusAttentionDecoderLayer(
)
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
declare_attn(
read=NormQuantRead(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": False,
},
),
(
declare_ffn(
@@ -793,7 +788,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
input_deepstack_embeds[:, start : start + self.hidden_size],
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -607,7 +607,7 @@ class LagunaModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
+1 -1
View File
@@ -761,7 +761,7 @@ class LLaDA2MoeModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
+1 -1
View File
@@ -535,7 +535,7 @@ class Llama4Model(nn.Module):
else None,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+3 -3
View File
@@ -448,16 +448,16 @@ class LongcatFlashDecoderLayer(nn.Module):
)
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
(
declare_ffn(ordinary_only=True),
declare_ffn(),
self.post_attention_layernorm[0],
),
(
declare_attn(ordinary_only=True),
declare_attn(),
self.input_layernorm[1],
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
),
(
declare_ffn(ordinary_only=True),
declare_ffn(),
self.post_attention_layernorm[1],
),
prepared_from=self.moe_stage.declaration,
@@ -272,7 +272,7 @@ class LongcatFlashModelNextN(nn.Module):
positions, hidden_states, forward_batch, zero_allocator
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
+1 -1
View File
@@ -1069,7 +1069,7 @@ class MiMoV2Model(nn.Module):
else None,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
# A draft targeting the final layer ("after layer
# num_hidden_layers-1") maps to capture index num_hidden_layers,
+1 -1
View File
@@ -228,7 +228,7 @@ class MiMoV2ModelNextN(nn.Module):
forward_batch=forward_batch,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
if forward_batch.return_hidden_states_before_norm:
+1 -1
View File
@@ -1176,7 +1176,7 @@ class MiniMaxM2Model(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -1517,7 +1517,7 @@ class MiniMaxM3Model(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -753,7 +753,7 @@ class NemotronHModel(nn.Module):
return residual_batch.to_pp(
hidden_states, forward_batch, preserve_declared=True
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
residual_batch.snapshot(hidden_states, forward_batch)
+13 -20
View File
@@ -1,15 +1,16 @@
"""Layer-communication helpers for the Nemotron-H model."""
from typing import Optional
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.communicator import (
StageKind,
HandoffRows,
ProducerReduction,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.runtime_context import get_parallel
ATTN_LAYERS = (MAMBA, ATTENTION)
@@ -18,33 +19,23 @@ def is_attn_layer(layer_type: str) -> bool:
return layer_type in ATTN_LAYERS
def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]:
"""Which stage of a decoder layer the stage at ``layer_idx`` stands for: a
Mamba or attention mixer the attention, an MLP or MoE the FFN; None past
either end."""
if not 0 <= layer_idx < len(pattern):
return None
return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN
def _declaration(pattern: str, layer_idx: int):
following = _stage_kind(pattern, layer_idx + 1)
if is_attn_layer(pattern[layer_idx]):
return declare_attn(
mixer_exit=True,
next_kind=following,
ordinary_only=True,
read=NormQuantRead(before_gather=True),
reduction=ProducerReduction.SCOPED,
gathers_tp_input=False,
)
return declare_ffn(
sparse=pattern[layer_idx] == MOE,
ordinary_only=True,
return_to_attention=True,
read=NormRead(before_gather=True),
dense_tp_size=get_parallel().tp_size if pattern[layer_idx] != MOE else None,
handoff_rows=HandoffRows.ATTENTION,
)
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
from sglang.srt.layers import layernorm_sp
from sglang.srt.runtime_context import get_parallel
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
@@ -53,9 +44,11 @@ def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
(
_declaration(pattern, layer_idx),
layer_norm,
{"force_layernorm_before_dp_gather": True},
),
previous=previous,
following=_declaration(pattern, layer_idx + 1)
if layer_idx + 1 < len(pattern)
else None,
terminal=layer_idx == len(pattern) - 1,
)
return boundary
+2 -3
View File
@@ -1214,7 +1214,7 @@ class Qwen2MoeModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
@@ -1310,14 +1310,13 @@ class Qwen2MoeForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states = residual_batch.complete_output(
forward_batch.hidden_states, forward_batch
)
# norm
hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
+1 -1
View File
@@ -477,7 +477,7 @@ class Qwen3Model(Qwen2Model):
)
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return (
+18 -41
View File
@@ -29,6 +29,8 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
qwen3_5_gdn_prefill_projection_views,
)
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
# Configs
from sglang.srt.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5MoeConfig,
@@ -52,12 +54,13 @@ from sglang.srt.layers.communicator import (
# Layers - Attention
from sglang.srt.layers.communicator.residual import batch as residual_batch
# Layers - Others
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
# Layers - Others
from sglang.srt.layers.layernorm import GemmaRMSNorm
# Layers - Linear
@@ -259,32 +262,6 @@ if _is_npu:
from sgl_kernel_npu.fla.utils import fused_qkvzba_split_reshape_cat_contiguous
@lru_cache(maxsize=1)
def _enable_qwen35_fused_ar_quant() -> bool:
"""Gate the fused AR+RMSNorm+per-group-FP8-quant path for Qwen3.5.
The single-kernel backend is ROCm/aiter/gfx95-only. The model gate stays
tied to ROCm/aiter so non-gfx95 HIP can keep the existing 2-kernel fallback
behavior for tuple handoff when this branch is used. It replaces the
existing ``--enable-aiter-allreduce-fusion`` 3-kernel path
(AR → RMSNorm → per-group quant) with either a single fused kernel (when
the fully-fused variant is eligible) or a 2-kernel path
(fused AR+RMSNorm + separate per-group quant) that still saves one
kernel launch vs. baseline. The stage boundary gracefully falls back
to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused
quant helper returns ``None``, so turning this on never regresses the
AR+RMSNorm fusion itself.
Opt-out: set ``SGLANG_DISABLE_FUSED_AR_QUANT=1`` to fall back to the
unmodified AR+RMSNorm fusion path.
"""
if not _use_aiter:
return False
if get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false"):
return False
return bool(get_exec().comm.enable_aiter_allreduce_fusion)
def _linear_accepts_fp8_tuple(linear: nn.Module) -> bool:
quant_method = getattr(linear, "quant_method", None)
return quant_method.__class__.__name__ == "Fp8LinearMethod" and (
@@ -1115,18 +1092,17 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
# GDN layers need both bf16 (for the small in_proj_ba gating
# projection) and a quantized tuple only when in_proj_qkvz can consume
# it. Otherwise, stay on the plain AR+RMSNorm path.
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant()
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
declare_attn(
read=NormQuantRead(
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
"fusions": boundary_fusions,
},
),
@@ -1322,17 +1298,17 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
# Standard attention layers benefit from a fused quant epilogue only
# when qkv_proj can consume the returned quantized tuple.
enable_fused_ar_quant = (
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
)
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
boundary_fusions = _layer_fusions(config, is_nextn)
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
declare_attn(
read=NormQuantRead(
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
)
),
self.input_layernorm,
{
"enable_fused_ar_quant": enable_fused_ar_quant,
"fused_ar_quant_keep_bf16": False,
"fusions": boundary_fusions,
},
),
@@ -1752,6 +1728,7 @@ class Qwen3_5ForCausalLM(nn.Module):
# Every layer was checked above.
can_defer_finalize=lambda layer: True,
label="Qwen3.5",
# The final GemmaRMSNorm consumes a finalize handoff.
terminal_finalize=True,
)
+1 -2
View File
@@ -1099,14 +1099,13 @@ class Qwen3MoeForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states = residual_batch.complete_output(
forward_batch.hidden_states, forward_batch
)
# norm
hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
+1 -1
View File
@@ -943,7 +943,7 @@ class Qwen3NextModel(nn.Module):
else None,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -1267,7 +1267,7 @@ class Qwen3LLMModel(Qwen3Model):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states,
+1 -1
View File
@@ -157,7 +157,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states,
+3 -3
View File
@@ -1169,7 +1169,7 @@ class SarvamMLAModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
@@ -1272,7 +1272,7 @@ class SarvamMLAForCausalLM(nn.Module):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states = residual_batch.complete_output(
forward_batch.hidden_states, forward_batch
)
forward_batch.hidden_states = residual_batch.norm(
@@ -1451,7 +1451,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states = residual_batch.complete_output(
forward_batch.hidden_states, forward_batch
)
forward_batch.hidden_states = residual_batch.norm(
+1 -1
View File
@@ -412,7 +412,7 @@ class SDARModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
+1 -1
View File
@@ -478,7 +478,7 @@ class SDARMoeModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -478,7 +478,7 @@ class Step3TextModel(nn.Module):
layer = self.layers[i]
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
+1 -1
View File
@@ -711,7 +711,7 @@ class Step3p5Model(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
+1 -1
View File
@@ -116,7 +116,7 @@ class Step3p5AMultiTokenPredictor(nn.Module):
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
hidden_states_before_norm = None
if not forward_batch.forward_mode.is_idle():
# if forward_batch.return_hidden_states_before_norm:
+1 -1
View File
@@ -1646,7 +1646,7 @@ class XllmModel(nn.Module):
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
+71 -32
View File
@@ -2,12 +2,13 @@
from types import SimpleNamespace
from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan
from sglang.srt.layers.communicator.construction import StagePlan
from sglang.srt.layers.communicator.factories import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.ops import identity_output
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
from sglang.srt.layers.communicator.stage import StageCommunicator
@@ -40,20 +41,14 @@ def make_test_stages(
next_sparse=next_sparse,
read=residual.ffn_read,
update=residual.ffn_update,
output=output,
output_transform=output,
)
common = {
key: options.pop(key)
for key in ("force_layernorm_before_dp_gather", "fusions")
if key in options
}
common = {key: options.pop(key) for key in ("fusions",) if key in options}
attn, ffn = make_stages(
(
attention,
attention_norm,
dict(
residual_in_hidden=residual.ffn_update.at_producer, **common, **options
),
dict(**common, **options),
),
(ffn, ffn_norm, common),
previous=previous,
@@ -69,17 +64,8 @@ def stub_plan():
plan.fusions = None
plan._publish_lora_layout = False
plan._fusion_rows = None
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
plan.is_first_layer = False
plan.is_last_layer = False
plan.residual_in_hidden = False
# Keep production variant selection while tests replace individual paths.
plan._batch_steps = lambda batch: {
BatchVariant.ORDINARY: plan._steps,
BatchVariant.CONTEXT_PARALLEL: plan._cp_steps,
BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps,
BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps,
}[plan._variant(batch)]
plan._paths = {}
plan.enters_stack = False
return plan
@@ -87,7 +73,10 @@ def stub_stage(plan, kind):
"""Keep the same concrete boundary when a test patches its methods."""
stages = plan.__dict__.setdefault("_stub_stages", {})
if kind not in stages:
stages[kind] = StageCommunicator(plan, kind, plan.norm)
stages[kind] = StageCommunicator(
plan,
declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()),
)
return stages[kind]
@@ -95,18 +84,16 @@ def sp_region_steps():
"""Local SP rows with no owed sum, for tests of activation and exits."""
from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
NORM_READ,
BoundarySteps,
CommunicateSummableTensorPairFn,
EdgeDecl,
Layout,
StageEntry,
StageInput,
StageOutput,
StageSteps,
TokenAxis,
make_boundary,
)
from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input
from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
output = StageOutput(rows)
@@ -117,10 +104,62 @@ def sp_region_steps():
)
return StageEntry(selected.prepare, rows, handoff=handoff)
return BoundarySteps(
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input),
entry(NORM_READ),
output,
CommunicateSummableTensorPairFn._trivial,
False,
return StageSteps(
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input), output, identity_output
)
def prepare_input(stage, hidden, residual, forward_batch, **call):
return prepare_raw(stage, "_prepare_input", hidden, residual, forward_batch, **call)
def prepare_attention(stage, hidden, residual, forward_batch, *args, **call):
return prepare_raw(
stage, "_prepare_attention", hidden, residual, forward_batch, *args, **call
)
def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.communicator.residual.stream import ResidualStream
if not isinstance(residual, ResidualStream):
stream = ResidualStream(residual)
if residual is not None:
hidden = stream.leave(
hidden,
call.get("update", ADD),
declared_sum=stage.entry(forward_batch).input_sum,
)
else:
stream = residual
return getattr(stage, method)(hidden, stream, forward_batch, *args, **call)
def finish_exit(scope, hidden, residual):
from sglang.srt.layers.communicator.residual.stream import ResidualStream
if isinstance(residual, ResidualStream):
scope._stream = residual
return scope.finish(hidden), residual
scope._stream.residual = residual
output = scope.finish(hidden)
if scope._stream.pending is None:
return output, None
return scope._stream.input(output)
def postprocess_output(boundary, hidden, residual, forward_batch):
from sglang.srt.layers.communicator.residual.stream import ResidualStream
stream = (
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
)
hidden = boundary.postprocess_layer(hidden, stream, forward_batch)
if isinstance(residual, ResidualStream):
return hidden, stream
return (hidden, None) if stream.pending is None else stream.input(hidden)
def identity_input(hidden_states, forward_batch):
return hidden_states
+6 -5
View File
@@ -12,19 +12,19 @@ from sglang.srt.layers import communicator as communicator
from sglang.srt.layers.communicator import (
boundary,
construction,
contracts,
exit,
factories,
layer,
layout,
legacy_stage,
ops,
output,
prepare,
)
from sglang.srt.layers.communicator import residual as residual_contract
from sglang.srt.layers.communicator import (
stage,
)
from sglang.srt.layers.communicator.adapters import attention, branch, lora
from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap
from sglang.srt.layers.communicator.fusions import allreduce
from sglang.srt.layers.communicator.residual import (
access,
@@ -40,6 +40,7 @@ COMMUNICATOR_MODULES = (
exit,
branch,
lora,
overlap,
layout,
output,
attention,
@@ -50,10 +51,10 @@ COMMUNICATOR_MODULES = (
ops,
boundary,
construction,
contracts,
prepare,
factories,
stage,
layer,
legacy_stage,
batch,
stream,
)