[Refactor] Document layer boundary contracts and integration (#41557)

This commit is contained in:
Cheng Wan
2026-09-28 20:18:27 -07:00
committed by GitHub
parent 49e7bd179d
commit 6dad152490
35 changed files with 961 additions and 132 deletions
+4 -3
View File
@@ -626,9 +626,10 @@ class ExecComm(msgspec.Struct):
help="Select FFN boundary reduction: ar uses all-reduce then token "
"redistribution; rs and rsv permit fixed-size and variable-size "
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
"Unsupported paths fall back to ar. auto resolves the model default "
"for all FFN layers. Required attention and MoE collectives and "
"all-reduce fusion are unaffected.",
"Unsupported paths fall back to ar. The option applies to the FFN "
"stages of decoders built with stage boundaries, where auto resolves "
"the model default; other models ignore it. Required attention and "
"MoE collectives and all-reduce fusion are unaffected.",
resolvable=True,
),
] = "auto"
@@ -422,7 +422,7 @@ class MambaMixer2(torch.nn.Module):
set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# By default a layer communicator reduces the output under DP attention.
# By default the stage boundary reduces the output under DP attention.
if reduce_results is None:
reduce_results = not is_dp_attention_enabled()
self.out_proj = RowParallelLinear(
@@ -11,4 +11,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""What the communicator takes from the modules it serves."""
"""What the stage boundaries take from the modules they serve."""
@@ -57,7 +57,7 @@ class AttentionInputs:
self.hidden_states_ = None
self.qkv_latent_ = None
# When True, hidden_states_local is already attn_tp-gathered upstream
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
self.is_pre_gathered = is_pre_gathered
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
@@ -156,11 +156,19 @@ def make_boundary(
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> Boundary:
"""Bind only the update capabilities that can reach this edge.
"""Bind the consumer half of an edge at construction time.
Execution receives the producer's actual update from the residual stream.
A single capability needs no dispatcher; multiple capabilities select one
of the already constructed paths at the consumer.
Args:
edge: Producer/consumer contracts, residual rows and update capabilities.
fusions: Ordered candidates for completing a declared sum with the read.
carried_fusions: Ordered candidates accepting dynamically carried work.
cp_moves: Context-parallel strategy operations for this edge, if needed.
enters_stack: Whether the read must initialize the stack's residual.
Returns:
A Boundary whose prepare accepts the actual update from the stream.
Multiple update capabilities select among preconstructed paths; a single
capability needs no runtime dispatcher.
"""
capabilities = edge.update_capabilities
if not capabilities:
@@ -287,10 +295,18 @@ def _bind_consumer(
def make_output_boundary(
edge: EdgeDecl, *, cp_moves: Optional[CpMoves] = None
) -> Boundary:
"""The producer's half of ``edge``, out of a layer's last stage onto the
rows the layer hands on (``edge.need``), whose consumer runs in the next
layer: the postprocess that moves the output there. ``cp_moves`` for an
edge that returns across attention CP."""
"""Bind the producer half of a layer/branch handoff.
Args:
edge: Output contract and destination/residual rows. Deferred updates
must outlive the producer; pipeline handoffs require a plain add
or a residual already written by the producer.
cp_moves: Context-parallel return operations, when the edge needs them.
Returns:
A Boundary with fixed transport or a marker for batch-dependent DP
transport. The receiver binds its read independently.
"""
update = edge.produced.update
if not getattr(update, "at_producer", False):
if not getattr(update, "can_defer_across_layers", False):
@@ -108,10 +108,15 @@ def _input_can_be_scattered() -> bool:
@dataclass(frozen=True)
class StageEdges:
"""One stage's incoming and outgoing declarations for one batch variant.
"""One stage's incoming and outgoing contracts for one batch variant.
No adjacent stage object is required. Edges can come from a local model
assembler, pipeline contract or branch adapter.
Fields:
incoming: Consumer-side contract used to bind prepare.
outgoing: Producer-side contract used to bind exit transport.
handoff: Optional attention adapter run after input preparation/movement.
cp_moves: Strategy-specific context-parallel gather and return operations.
No executable neighbouring stage or neighbouring norm is required.
"""
incoming: EdgeDecl
@@ -125,7 +130,26 @@ def _requires_branch_input(*args, **kwargs):
class StagePlan:
"""Precomputed paths for one stage; never owns the neighbouring norm."""
"""Bind reusable entry/exit paths from a stage's resolved declarations.
Args:
kind: Boundary role selecting attention or FFN fusion adapters.
norm: This stage's consumer normalization module.
variants: Mapping from BatchVariant to StageEdges. Construction binds
each entry once; a forward selects an existing path by batch facts.
enters_stack: Whether this stage initializes the residual from embeddings.
prepared_input: Branch reusing an already-read input; requires branch_input
instead of a normal prepare to avoid repeating the read/update.
terminal: Whether the stage ends the model's layer stack.
direct_handoff: Attention can publish its output with finish instead of
an exit scope and output transport.
qkv_latent_func: Optional hook for prepared attention input.
fusions: Optional backend provider of ordered consumer fusion candidates
and the producer deferral policies (see make_attn_stage).
The plan owns static paths, not per-forward tensors or a neighbour's norm.
Runtime residual state belongs to the ForwardBatch's ResidualStream.
"""
def __init__(
self,
@@ -27,7 +27,14 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
class ProducerReduction(Enum):
"""Whether compute always leaves its sum or follows the boundary scope."""
"""Describe compute's cooperation with its output boundary.
PARTIAL is attention-only: finish() publishes its partial sum. SCOPED
follows exit() flags for an FFN or single-stage mixer. LOCAL_TAIL is FFN-only:
compute adds a replicated component after its internal sum, so the
boundary never defers that sum to the next layer. A selected reduce-scatter
still applies, so compute must add the tail only when it completes the sum.
"""
PARTIAL = auto()
SCOPED = auto()
@@ -50,8 +57,14 @@ class BatchVariant(Enum):
class StageInput(msgspec.Struct, frozen=True):
"""The rows a stage's consumer needs: sharded over the token axes its
compute group does not span."""
"""Declare the consumer's input rows and read operation.
Fields:
layout: Token sharding required by compute.
gathers_itself: Token axes compute can gather internally; the boundary
may hand it input still sharded over those axes.
read: Operation that derives input from the updated residual.
"""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
@@ -62,14 +75,25 @@ class StageInput(msgspec.Struct, frozen=True):
class StageOutput(msgspec.Struct, frozen=True):
"""What a stage's producer hands the boundary after it, fixed at
construction: its rows, the group its output is summed over, and when it
leaves that sum to the boundary instead of completing it.
"""Declare producer rows and permitted reduction handoffs at construction.
What one output actually owes is carried with the value: a plain
tensor handed across layers is complete, an ``UnreducedOutput`` names what
is left. A raw compute output entering a boundary is read with this
declaration and the boundary's decision for the batch."""
Fields:
layout: Token sharding of the producer contribution.
group: Named sum group, or None when there is no reduction.
always_leaves: Compute always returns a partial sum.
leaves_for_next_layer: Compute can skip reduction under the exit scope
and let the following layer complete it.
leaves_for_reduce_scatter: Compute can leave reduction to a fixed-size
reduce-scatter selected by the boundary.
leaves_for_reduce_scatterv: Compute can leave reduction to the selected
variable-size attention-DP combine.
update: Producer residual operation; None on an arrival contract that
declares capabilities and obtains the actual update from the stream.
transform: Optional operation on the contribution before residual update.
These permissions are not evidence that a particular output is partial.
The exit decision and ResidualStream record what that output actually owes.
"""
layout: Layout
# None when there is nothing to sum.
@@ -81,8 +105,7 @@ class StageOutput(msgspec.Struct, frozen=True):
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
leaves_for_next_layer: bool = False
leaves_for_reduce_scatter: bool = False
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
# Whether the selected attention-DP combine may complete this sum.
leaves_for_reduce_scatterv: bool = False
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
@@ -91,17 +114,31 @@ class StageOutput(msgspec.Struct, frozen=True):
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
"""Resolved input and output contracts for one stage and batch variant.
Fields:
input: Consumer row requirement and read operation.
output: Producer row, reduction and residual-update contract.
"""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
rows the residual is on before and after the boundary."""
"""Describe both sides of a boundary and the residual's row movement.
Fields:
produced: Producer output contract as seen by this side of the edge.
need: Consumer input contract.
residual: Residual layout before the boundary.
residual_to: Residual layout after the boundary.
residual_joins_sum: Whether one rank may add the residual into a partial
before reduction; valid only for an eligible plain-add update.
update_capabilities: Allowed values of StageUpdate.adds_plainly for
arriving contributions. Empty means use produced.update's capability.
The actual update object travels with the residual stream.
"""
produced: StageOutput
need: StageInput
@@ -145,7 +182,27 @@ class CpMoves(msgspec.Struct, frozen=True):
class StageEntry(msgspec.Struct, frozen=True):
"""The boundary into one of a layer's stages, as the layer runs it."""
"""Bound consumer operations for one batch variant.
Fields:
prepare: Callable(hidden_states, residual, forward_batch, norm, **call)
returning compute input and updated residual; finishes owed work,
applies the producer update and performs the consumer read.
input_rows: Layout handed to compute after preparation and input_move.
input_move: Optional movement after prepare, before the compute handoff.
handoff: Optional callable(input, forward_batch, qkv_latent_func) that
adapts already-placed input for attention.
capture_move: Optional movement of the updated residual back onto the
producer's rows for auxiliary capture.
capture_move_allocates: Whether capture_move returns fresh storage (a
gather) rather than a view (a cut); capture then retains it
without copying.
input_sum: Statically owed sum group for an otherwise raw input tensor.
preserves_residual: Optional (residual, batch) predicate certifying this
input path leaves residual untouched, including backend fallback.
capture_preserves_residual: Same guarantee for the local following stage,
copied at construction so auxiliary capture can retain its input.
"""
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
@@ -169,7 +226,16 @@ class StageEntry(msgspec.Struct, frozen=True):
class StageSteps(msgspec.Struct, frozen=True):
"""One batch variant of one stage: its input and output boundary paths."""
"""Precomputed entry and exit work for one stage and batch variant.
Fields:
entry: Bound consumer preparation and handoff.
output: Producer contract used by the exit decision.
output_move: Fixed output transport, or None when absent or chosen per
batch by the attention-DP exit path.
output_move_completes_sum: Whether that move also reduces the output.
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
"""
entry: StageEntry
output: StageOutput
@@ -181,9 +247,11 @@ class StageSteps(msgspec.Struct, frozen=True):
class StageKind(Enum):
"""Which of a decoder layer's two stages a layer that is one stage takes
the place of: its norm, its read and update, its fused kernels and its
entry in the layer's steps."""
"""Select attention/mixer or FFN input/output adapters.
This is a boundary role, not a compute implementation or a requirement
that layers contain exactly two alternating stages.
"""
ATTENTION = auto()
FFN = auto()
@@ -344,7 +344,9 @@ class OutputBoundary:
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
"""Admit ordinary single-sum outputs, plus LoRA and TP1 shared-expert
outputs when a fused consumer (the backend's can_defer_all_reduce,
FlashInfer or aiter) can take them.
A fused-kernel fallback completes the partial-output contract selected
before the producer ran, without recomputing the producer's LoRA path.
@@ -379,9 +381,18 @@ def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool
class FfnCompletion(msgspec.Struct, frozen=True):
"""One FFN's reduction decision. The flags are published while the FFN runs;
``complete(hidden_states, residual)`` then either wraps the output for the
next layer's input to complete, or runs this layer's postprocess step."""
"""One decision shared by compute flags and the matching output completion.
Fields:
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later
or carries it to the next consumer, whether fused there or unfused.
mlp_reduce_scatter: Compute leaves reduction to the selected scatter path.
complete: Callable(output, residual) returning the completed or wrapped
output and its corresponding residual rows.
Do not independently reselect completion after compute has used these flags.
"""
defer_moe_finalize: bool
fuse_mlp_allreduce: bool
@@ -402,8 +413,10 @@ class MixerExit:
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
row-parallel output projection to skip the all-reduce. It skips when the
stage's output always leaves its sum (to an FFN stage, which completes it in
its input), and when its output declaration permits deferring the sum. The
consumer decides whether to fuse its completion with the input norm."""
its input), or when its declaration permits deferring the sum to the next
attention stage and the batch allows it: TP > 1, no attention DP, and no
input-scattered or MoE-CP all-gather layout. The consumer decides whether
to fuse its completion with the input norm."""
__slots__ = (
"skips_reduction",
@@ -454,9 +467,22 @@ class MixerExit:
class FfnExit:
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
``defer_moe_finalize`` on ``get_forward()``."""
"""Scope a selected FFN output decision and retain its completion action.
Args:
boundary: OutputBoundary that owns the producer's bound paths.
forward_batch: Batch selecting reduction/finalize eligibility.
stream: This invocation's residual stream, updated by finish().
Fields:
boundary: The owning output boundary.
defer_moe_finalize: Whether compute may return a finalize handoff.
fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion.
mlp_reduce_scatter: Whether compute leaves reduction to a scatter path.
The three flags are published on get_forward() only inside the context.
finish(output) uses the same selected action after successful compute.
"""
__slots__ = (
"boundary",
@@ -186,11 +186,32 @@ def _ffn_decl(
@dataclass(frozen=True)
class StageDeclaration:
"""A computation and its input source, without an execution plan.
"""Describe one computation's boundaries without binding kernels or norms.
``previous`` consumes a producer's output. ``prepared_from`` reuses an
already-read input for a branch; the two sources are mutually exclusive.
Sources may be reconstructed locally, including across pipeline ranks.
Fields:
kind: Selects attention/mixer or FFN boundary adapters; does not run
the corresponding computation.
read: Consumer operation that derives compute input from the residual.
update: Producer operation that writes compute output into the residual.
sparse: Whether this FFN is a MoE; used to resolve input/output rows.
Expert routing and all-to-all remain inside the MoE computation.
terminal: Whether this stage ends the model's layer stack. Prevents
leaving work that requires a following layer; a finalize handoff
may still reach the terminal norm when the fusion provider allows it.
output_transform: Optional operation on the FFN contribution before
residual update, with an explicit reduction-order contract.
reduction: Whether compute always leaves a partial sum, obeys the
exit scope, or adds a replicated component after its own sum.
gathers_tp_input: Whether attention gathers TP-sharded input itself.
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
handoff_rows: Required FFN output rows at the layer or branch handoff.
previous: Declaration whose output this stage consumes. It may be
reconstructed locally, including across pipeline ranks.
prepared_from: Declaration whose already-read input a branch reuses.
Mutually exclusive with previous; avoids a second update/read.
Sources contain declarations, never executable boundary objects.
"""
kind: StageKind
@@ -217,10 +238,16 @@ class StageDeclaration:
@dataclass(frozen=True)
class StageConnection:
"""The producer's exit and consumer's entry for each batch variant.
"""Immutable, norm-free contracts for the two sides of a connection.
Immutable, norm-free declarations. Independently constructed equivalent
connections work across pipeline partitions; object identity is irrelevant.
Fields:
producer: Source declaration, or None at stack entry.
consumer: Destination declaration, or None at a layer/stack handoff.
exits: Producer-side EdgeDecl for each supported BatchVariant.
entries: Consumer-side EdgeDecl for each supported BatchVariant.
Equivalent declarations can be reconstructed across pipeline partitions;
adjacent layers need not share the same connection object.
"""
producer: Optional[StageDeclaration]
@@ -243,6 +270,22 @@ def declare_attn(
reduction=ProducerReduction.PARTIAL,
gathers_tp_input=True,
):
"""Declare attention or a mixer; construct its executable boundary later.
Args:
previous: Producer declaration whose output this stage consumes.
prepared_from: Source of an already-read branch input, instead of previous.
read: Input operation; defaults to normalization with quantization support.
update: Operation that adds this stage's output to the residual.
terminal: Whether this stage ends the model's layer stack.
reduction: PARTIAL for an output projection that always skips reduction;
SCOPED for a mixer that follows its exit scope's reduction decision.
LOCAL_TAIL is rejected for attention stages.
gathers_tp_input: Whether compute gathers attention-TP input slices itself.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
"""
if reduction is ProducerReduction.LOCAL_TAIL:
raise ValueError("LOCAL_TAIL is not supported for attention stages")
return StageDeclaration(
@@ -271,6 +314,29 @@ def declare_ffn(
reduction=ProducerReduction.SCOPED,
handoff_rows=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
Args:
previous: Producer declaration whose output this stage consumes.
prepared_from: Source of an already-read branch input, instead of previous.
sparse: Whether the FFN is a MoE; EP dispatch stays inside compute.
read: Operation deriving FFN input from the residual.
update: Operation writing FFN output into the residual.
terminal: Whether this stage ends the model's layer stack.
output_transform: Optional contribution transform before residual update.
next_sparse: Whether the next decoder layer's FFN is sparse; used only
to derive the TBO handoff when handoff_rows is not supplied.
dense_tp_size: Dense compute width: None for configuration, 1 for local
compute, or the full TP size.
reduction: How compute cooperates with the exit's reduction decision:
SCOPED (default) follows the exit scope; LOCAL_TAIL marks a
replicated tail after the sum. PARTIAL is rejected for FFN stages.
handoff_rows: Explicit output-row requirement; otherwise derived from
the adjacent FFN kinds and TBO configuration.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
"""
if reduction is ProducerReduction.PARTIAL:
raise ValueError("PARTIAL is not supported for ffn stages")
return StageDeclaration(
@@ -522,11 +588,23 @@ def make_attn_stage(
qkv_latent_func=None,
fusions=None,
):
"""Resolve this attention's boundaries, then bind its norm and hooks.
"""Resolve one attention/mixer's boundaries and bind its input norm.
``following`` supplies the consumer declaration for a stage inside a
layer. With no following stage, the output is handed across a layer or
stack boundary; the receiver binds its own read independently.
Args:
declaration: Attention StageDeclaration, including its input source.
norm: This consumer's normalization module, never its neighbour's norm.
following: Local consumer declaration whose previous is declaration.
None leaves a layer/stack handoff for an independently bound reader.
qkv_latent_func: Optional attention input hook, invoked after preparation
and movement onto the compute input rows.
fusions: Optional backend provider. Consumer side: ordered
attention_input(plan) and ffn_input(plan) candidates. Producer side
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
and can_defer_all_reduce(plan, batch), called when LoRA or TP1 shared
experts are enabled.
Returns:
A StageBoundary with precomputed paths for supported batch variants.
"""
if declaration.kind is not StageKind.ATTENTION:
raise TypeError("make_attn_stage requires an attention declaration")
@@ -548,7 +626,18 @@ def make_ffn_stage(
following: Optional[StageDeclaration] = None,
fusions=None,
):
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
"""Resolve one FFN's boundaries and bind its input norm.
Args:
declaration: FFN StageDeclaration, including its input source.
norm: This FFN's input normalization module.
following: Local consumer declaration whose previous is declaration;
None leaves a layer/stack handoff for an independently bound reader.
fusions: Optional backend fusion provider, as in make_attn_stage.
Returns:
A StageBoundary. Expert routing and all-to-all stay inside compute.
"""
if declaration.kind is not StageKind.FFN:
raise TypeError("make_ffn_stage requires an FFN declaration")
incoming, outgoing = _connections(declaration, following)
@@ -564,18 +653,23 @@ def make_ffn_stage(
def make_stages(
*stages, previous=None, prepared_from=None, following=None, terminal=False
):
"""Bind a local linear sequence and return its independent boundaries.
"""Bind a local linear sequence of any positive number of stages.
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
Options are the stage-specific keyword arguments of ``make_attn_stage``
or ``make_ffn_stage``; they are consumed only during construction.
Declarations in the sequence have no input source: this function connects
them in order. Only the final boundary is terminal when requested.
Args:
*stages: Items of (declaration, norm) or (declaration, norm, options).
Declarations must have no source or terminal flag. Options are
constructor keywords: fusions, and qkv_latent_func for attention.
previous: External producer declaration consumed by the first stage.
prepared_from: Already-read input declaration reused by the first stage
of a branch; mutually exclusive with previous.
following: External consumer declaration after the last local stage.
None denotes a layer/stack handoff with an independently bound read.
terminal: Marks only the final stage as the end of the model's stack.
``previous`` describes an external producer's output. ``prepared_from``
describes an already-read branch input, obtainable from the source
boundary's immutable ``declaration``. Neither takes an executable boundary.
No sequence object or runtime routing is retained.
Returns:
A tuple of independent StageBoundary objects in declaration order.
Sources are connected on copied declarations; caller inputs are unchanged.
No sequence object or runtime routing is retained.
"""
if not stages:
raise ValueError("make_stages needs at least one stage")
@@ -1,8 +1,9 @@
"""FlashInfer MNNVL CuTe DSL AllReduce fusion, shared across architectures.
Two patterns share one workspace, both consumed at the next layer's input
RMSNorm: AR + residual + RMSNorm, and the same with the MoE finalize and the
shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
RMSNorm (and, with terminal_finalize, at the final norm): AR + residual +
RMSNorm, and the same with the MoE finalize and the shared-expert add folded in
when the runner hands back a MoeFinalizeHandoff.
"""
from __future__ import annotations
@@ -201,7 +202,10 @@ class CuteDSLFusion:
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
handing off a deferred MoE finalize with its unfused completion.
install_cutedsl_fusion supplies the service and producer capability;
consumer eligibility is checked only where the output is consumed."""
consumer eligibility is checked only where the output is consumed. Producer
policy: can_defer_all_reduce keeps a LoRA/TP1 shared-expert sum deferrable
while the workspace can take it; can_defer_finalize also covers a terminal
FFN when terminal_finalize is installed."""
def __init__(self) -> None:
self.service: CuteDSLFusionService | None = None
@@ -375,7 +379,10 @@ def install_cutedsl_fusion(
) -> CuteDSLFusionService | None:
"""One shared workspace handle per fusion-enabled layer, or None.
Every entry of ``layers`` carries its attention and FFN stages.
Every entry of ``layers`` carries ``attn_boundary`` and ``ffn_boundary``.
terminal_finalize lets the last layer hand its finalize to the final norm;
the model must then pass the returned service as
residual_batch.norm(handoff_norm=...).
"""
if get_flags().moe.in_speculative_scope:
# A draft shares the target's process, which holds one workspace.
@@ -48,8 +48,16 @@ class TokenAxis(Enum):
class Layout(msgspec.Struct, frozen=True):
"""The token axes a rank's rows are sharded over. Single-rank axes are left
out, so two layouts are equal exactly when every rank holds the same tokens."""
"""Describe token-row sharding under the active parallel topology.
Fields:
sharded: Axes across which ranks own different token rows. Factories
omit size-one axes through sharded_over().
This describes supported token partitions, not a general device mesh or
hidden-dimension placement. Row order and padding come from the batch's
shared token mapping, not from per-tensor metadata in this object.
"""
sharded: FrozenSet[TokenAxis]
@@ -399,7 +399,7 @@ def scatter_moe_cp_output(
forward_batch: ForwardBatch,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
@@ -22,11 +22,15 @@ from sglang.srt.distributed import GroupCoordinator
class OutputTransform(msgspec.Struct, frozen=True):
"""An operation on the producer contribution, before residual addition.
"""Transform a producer contribution before its residual update.
Some existing implementations apply it to a partial before reduce-scatter
but to a complete value after all-reduce. These are explicit alternatives,
not an assumption that an arbitrary operation commutes with reduction.
Fields:
apply: Callable(tensor) returning the transformed contribution.
before_reduce_scatter: Permit this operation on a partial before
reduce-scatter. Otherwise the transform follows completion.
This explicitly preserves an implementation's order; it does not imply
that arbitrary transforms commute with reduction or floating-point rounding.
"""
apply: Callable[[torch.Tensor], torch.Tensor]
@@ -34,10 +38,18 @@ class OutputTransform(msgspec.Struct, frozen=True):
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
it any other way. The producer says what is owed: one all-reduce over
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
"""Internal adapter value describing an unfinished reduction.
Fields:
partial: Tensor containing this rank's contribution to the sum.
group: All-reduce group when no redistribution callable is supplied.
reduce_and_redistribute: Callable(partial) completing the sum and moving
it to destination rows; takes precedence over group.
Exits hand this form to ResidualStream.leave(), which exposes an opaque
OwedOutput to models. Low-level adapters use reduce_output() before reading
it. A group is required when reduce_and_redistribute is absent.
"""
partial: torch.Tensor
group: Optional[GroupCoordinator] = None
@@ -48,7 +60,8 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
class HandoffOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes work only its producer knows how to do (a
MoE's finalize and sum), left for the next layer's input. A fused kernel
MoE's finalize and sum), left for the next layer's input or for a terminal
norm that accepts it (residual_batch.norm(handoff_norm=...)). A fused kernel
there may do that work together with its own; anything else passes it
through reduce_output(), which calls ``complete()``."""
@@ -436,8 +436,8 @@ def _mlp_input_gather_moe_cp(
update: StageUpdate = ADD,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
TP_ATTN_FULL."""
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays on
this rank's attention rows."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
@@ -22,7 +22,19 @@ import torch
class StageUpdate(Protocol):
"""How a stage's output is written into the residual."""
"""Producer-owned operation writing a contribution into the residual.
Fields:
adds_plainly: Whether update is plain addition, permitting compatible
add+norm fusion and adding the residual on one rank before a sum.
at_producer: Whether the producer writes the residual at its exit rather
than leaving the operation for the next prepare.
can_defer_across_layers: Whether parameters/state remain valid after
leaving the producer layer, including any offload or state reuse.
The shard conversion methods must move any update-associated state together
with the residual. Nonlinear updates must not claim adds_plainly.
"""
# A plain add, which one rank may run before the sum it adds into
# completes (the DP partial order, a residual that joins the attention
@@ -36,7 +48,12 @@ class StageUpdate(Protocol):
can_defer_across_layers: bool
def update(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the output written into it."""
"""Write hidden_states, the producer contribution, into residual.
Implementations define their residual representation and return the updated
tensor. Boundary ordering may put a plain add before an eligible sum.
This operation does not perform the consumer's normalization.
"""
def residual_to_attn_tp_shard(self, residual) -> torch.Tensor:
"""This attention-TP rank's slice of the residual, with whatever the
@@ -47,7 +64,18 @@ class StageUpdate(Protocol):
class StageRead(Protocol):
"""How a stage reads its input from the residual."""
"""Consumer-owned operation deriving compute input from the residual.
Fields:
norms_plainly: Read is normalization/optional quantization without changing
the residual, allowing compatible fused add+norm implementations.
before_gather: Preserve this read on source rows before a DP gather.
enter initializes the stack residual. read consumes an already-written
residual; update_and_read first applies the actual producer's update.
Both reads return (compute_input, residual), preserving their kernel's
rounding order rather than normalizing a separately rounded snapshot.
"""
# The input is the residual's norm, in the quantization the call asks for,
# and the residual is left as it is: what a fused add + norm kernel computes,
@@ -66,8 +94,19 @@ class StageRead(Protocol):
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple:
"""The input and the residual, from a residual that already holds the
previous stage's output."""
"""Read input from a residual that already includes the producer update.
Args:
residual: Updated residual tensor in this implementation's representation.
norm: Consumer normalization module.
quant_format: Requested input quantization format, or empty for the
read's default format; supported values belong to the implementation.
post_residual_addition: Kept for signature parity with update_and_read;
current read implementations ignore or reject it.
Returns:
(compute_input, residual), potentially with a quantized input format.
"""
def update_and_read(
self,
@@ -78,8 +117,20 @@ class StageRead(Protocol):
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple:
"""Write the previous stage's output into the residual with its
producer's ``update``, then read the input."""
"""Apply a producer update and read input while preserving kernel ordering.
Args:
update: Actual producer's StageUpdate, not a consumer-inferred operation.
hidden_states: Producer contribution after required communication.
residual: Previous residual, or None at entry where supported.
norm: Consumer normalization module.
quant_format: Requested input quantization format, as in read().
post_residual_addition: Optional extra added after update, before norm.
Returns:
(compute_input, residual). A fused implementation can preserve FP32
accumulation through norm without materializing a rounded intermediate.
"""
class LayerResidual(NamedTuple):
@@ -85,11 +85,13 @@ def finish_layer_stack(
final_norm_takes_handoff: bool = False,
preserve_declared: bool = False,
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers. A final norm
that does a producer's handoff together with its own work
(``final_norm_takes_handoff``) receives it as it is."""
"""Complete what this layer left for a next layer, for callers that carry
an explicit residual tensor or merge TBO microbatches. Stage-boundary
decoders use residual_batch.norm(), to_pp() or take_output() instead:
to_pp() keeps a declared partial sum for the receiving from_pp(), which
this helper would complete early. A final norm that does a producer's
handoff together with its own work (``final_norm_takes_handoff``)
receives it as it is."""
if isinstance(residual, ResidualStream):
return residual.finish(
hidden_states,
@@ -147,7 +147,7 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
and _is_flashinfer_available
and not is_dp_attention_enabled()
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
# cutedsl runs its own fused path from the fusion communicator.
# cutedsl runs its own fused path from its stage fusion (fusions/cutedsl.py).
and not uses_cutedsl_ar_fusion()
and not is_flashinfer_allreduce_unavailable()
# Symbolic size checks stay last: under Dynamo tracing they guard on
@@ -19,6 +19,11 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
def start(forward_batch):
"""Reset forward_batch.residual_stream at entry to a fresh layer-stack call.
Call once before the first local stage on an embedding path, not per layer.
Pipeline reception and TBO adapters reconstruct their own streams.
"""
forward_batch.residual_stream = ResidualStream()
@@ -54,6 +59,26 @@ def norm(
skip_empty=False,
**read_kwargs,
):
"""Complete the layer stack and apply its terminal normalization.
Args:
hidden_states: Current stream output or opaque owed handle.
forward_batch: Batch owning the residual stream.
layernorm: Final norm supporting the model's output/residual pair.
capture_output: Optional callback retaining the same updated residual;
it must copy borrowed storage when retention requires ownership.
handoff_norm: Optional adapter with finalize(handoff, residual, gamma)
for a producer-specific finalize handoff; when a handoff arrives it
cannot be combined with capture. Without it, a handoff is completed
unfused first. Its gamma is layernorm.gemma_weight, so the final
norm must then be a GemmaRMSNorm.
skip_empty: Skip the ordinary norm on a zero-row completed output.
**read_kwargs: Extra arguments forwarded to the two-input final norm.
Returns:
Normalized tensor. Add and norm remain together to preserve the kernel's
accumulation/rounding order; a snapshot is not used as the norm input.
"""
hidden_states, residual = current(forward_batch).finish(
hidden_states, takes_handoff=handoff_norm is not None
)
@@ -81,6 +106,17 @@ def norm(
def to_pp(hidden_states, forward_batch, *, preserve_declared=True):
"""Export hidden_states and residual as PPProxyTensors.
Args:
hidden_states: Current stream output or opaque owed handle.
forward_batch: Batch owning the stream.
preserve_declared: True (the default) sends a statically declared
partial sum unreduced for the receiver's from_pp to complete; pass
False only when the receiver does not declare that sum.
Runtime-selected completion work is finished before transport.
"""
hidden_states, residual = current(forward_batch).finish(
hidden_states, preserve_declared=preserve_declared
)
@@ -25,7 +25,13 @@ from sglang.srt.layers.layer_boundary.residual import StageUpdate
class CarriedSum(msgspec.Struct, frozen=True):
"""Work left by the producer, including a move back to local token rows."""
"""Runtime reduction left by an exit decision.
Fields:
group: All-reduce group when no redistribution callable is supplied.
reduce_and_redistribute: Optional callable(value) returning a reduced
value on destination rows; takes precedence over group.
"""
group: Optional[GroupCoordinator] = None
reduce_and_redistribute: Optional[Callable] = None
@@ -46,7 +52,15 @@ class DeclaredSum(msgspec.Struct, frozen=True):
class Contribution(msgspec.Struct):
"""The sole owner of an output's value, update and remaining work."""
"""Own a producer's output, residual update and outstanding completion.
Fields:
value: Contribution tensor, or None for an opaque finalize handoff.
update: Producer operation to apply before the consumer reads input.
owed: Runtime sum, declared sum, producer-specific finalize, or None.
Completing owed work clears owed but does not apply the residual update.
"""
value: Optional[torch.Tensor]
update: StageUpdate
@@ -80,11 +94,20 @@ class OwedOutput(msgspec.Struct, frozen=True):
class ResidualStream:
"""Uninitialized, written, or holding one pending producer contribution.
"""Own residual state for one forward or TBO microbatch.
Each forward or TBO microbatch owns its stream. A prepare consumes its
pending contribution once; completing a sum for capture leaves the update
pending. No layout or batch selection is stored on the value.
Args:
residual: Already-written residual, or None before stack entry.
Fields:
residual: Residual tensor retained across stages, or None initially.
pending: Contribution awaiting a residual update, or None. Its owed
field separately records incomplete reduction/finalize work.
The states are initial (both None), written (residual only), and pending.
Prepare consumes pending once and writes the resulting residual. Completing
communication for capture leaves the update pending. Layouts and batch-path
selection live on the bound stage, not on tensor values.
"""
__slots__ = ("residual", "pending")
@@ -95,7 +118,18 @@ class ResidualStream:
@classmethod
def arrive(cls, hidden, residual, update, *, declared_sum=None):
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
"""Reconstruct a stream from a tensor handoff, including a TBO microbatch.
Args:
hidden: Received output or already-written residual tensor.
residual: Separate residual tensor. None means hidden already holds
the written residual, rather than an uninitialized stack.
update: Producer operation for a separate contribution/residual pair.
declared_sum: Static sum owed by hidden when residual is separate.
Returns:
(handle, stream). Use start() for a fresh, uninitialized stack instead.
"""
stream = cls(residual)
if residual is None:
return stream.write(hidden), stream
@@ -107,6 +141,18 @@ class ResidualStream:
return residual
def leave(self, output, update, *, declared_sum=None):
"""Record one producer contribution without applying its residual update.
Args:
output: Complete tensor, UnreducedOutput, or producer-specific HandoffOutput.
update: Producer operation that the next boundary must apply.
declared_sum: SumGroup statically owed by a raw tensor, or None. Cannot
be combined with another output wrapper's completion contract.
Returns:
The tensor when complete, otherwise an opaque OwedOutput. Replacing an
unconsumed contribution is an error; retain the returned handle.
"""
if self.pending is not None:
raise RuntimeError("cannot replace an unconsumed producer contribution")
if declared_sum is not None:
@@ -174,7 +220,19 @@ class ResidualStream:
return value.clone() if self.residual is None else value + self.residual
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
"""Export the contribution/residual pair; keep the stream available to readers."""
"""Export the output/residual pair without closing or consuming the stream.
Args:
hidden: Current stream tensor or opaque owed handle.
takes_handoff: Pass a producer-specific finalize handoff through for a
terminal adapter that can consume it; otherwise complete it here.
preserve_declared: Export a raw declared partial sum for a receiver
whose incoming contract reconstructs that sum (for example PP).
Returns:
(output, residual), completing outstanding work unless explicitly
preserved. Does not apply the pending residual update.
"""
self.check(hidden)
if self.pending is None:
return hidden, None
@@ -46,8 +46,16 @@ if TYPE_CHECKING:
class StageBoundary:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
"""Model-facing preparation and completion for one compute stage.
Args:
plan: Precomputed entry/exit paths, this stage's norm and input hooks.
declaration: Immutable stage description used to construct related
boundaries without exposing this object's execution plan.
The boundary does not run attention or FFN computation. Per-forward residual
state is read from forward_batch.residual_stream, not stored here.
"""
def __init__(
self,
@@ -127,6 +135,24 @@ class StageBoundary:
return self.plan.input_on_attention_tp_slices
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
"""Finish the previous contribution and produce this stage's compute input.
Args:
hidden_states: Tensor or opaque owed-output handle belonging to this
forward's residual stream. Do not unwrap or modify a partial sum.
forward_batch: Batch owning the stream and selecting the prepared path.
cache: FFN-only NPU weight cache prefetched on the FFN input path.
**call: Attention-only read/adapter options: quant_format,
post_residual_addition, captured_last_layer_outputs and capture_output.
FFN stages do not accept these options. A capture callback is
called as capture_output(value, owned=bool): owned=True means
value is fresh storage it may keep, and owned=False means it
must copy before retaining.
Returns:
Compute input in the selected read's format (possibly quantized).
Updates forward_batch.residual_stream with the resulting residual.
"""
if cache is not None:
call["cache"] = cache
prepare = (
@@ -153,6 +179,17 @@ class StageBoundary:
)
def exit(self, forward_batch):
"""Select and scope one FFN or single-stage mixer's output decision.
Args:
forward_batch: Active batch whose stream receives the compute result.
Returns:
A context manager publishing reduction/finalize flags during compute.
Call its finish(output) exactly once after successful compute, including
when compute is skipped for an empty input. It completes or carries work
using the same decision rather than selecting a second path.
"""
result = (
self.plan.output.ffn_exit(forward_batch, stream=current(forward_batch))
if self.kind is StageKind.FFN
@@ -163,6 +200,18 @@ class StageBoundary:
return result
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
"""Restore the residual stream from a pipeline handoff.
Args:
tensors: PPProxyTensors containing hidden_states and, ordinarily, residual.
forward_batch: Batch receiving a new stream.
allow_missing_residual: Accept a handoff without the residual key for
model paths that explicitly support it.
Returns:
Tensor or owed handle for prepare. A producer-written residual and a
declared partial sum are reconstructed from the incoming contract.
"""
hidden_states, residual = from_pp(
tensors,
residual_in_hidden=(
@@ -181,6 +230,18 @@ class StageBoundary:
return current(forward_batch).snapshot(hidden_states)
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
"""Capture an output while preserving the caller's reduction timing.
Args:
hidden_states: Current stream output or opaque owed handle.
forward_batch: Batch owning that stream.
skip_empty: Return no snapshot for a known zero-row output buffer.
Returns:
(updated_handle, snapshot). Dynamic owed work is completed on the main
contribution; a declared sum is reduced only on the snapshot copy.
Always use updated_handle afterwards. Plain residual updates are required.
"""
stream = current(forward_batch)
if skip_empty:
storage = buffer(hidden_states)
@@ -191,6 +252,9 @@ class StageBoundary:
return hidden_states, stream.snapshot(hidden_states)
def postprocess(self, hidden_states, forward_batch):
"""Move an FFN output that compute already completed outside exit()
(the operation-scheduled TBO path) onto the rows the layer hands on;
it chooses no reduction step."""
return self.plan.output.postprocess_layer(
hidden_states, current(forward_batch), forward_batch
)
@@ -302,7 +302,7 @@ class BaseRunner(ABC):
# Nothing else resolves the configured backend, so check the platform.
resolve_flashinfer_allreduce_fusion_backend()
if not mr.is_draft_worker:
# A draft installs no fusion communicator.
# A draft installs no CuTe DSL fusion.
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
prepare_cutedsl_fusion,
)
+2 -2
View File
@@ -2633,7 +2633,7 @@ class DeepseekV2DecoderLayer(nn.Module):
qkv_latent_func: Optional[Callable],
output=None,
):
"""The communicator for this layer's norms; it chooses its boundary
"""The stage boundaries for this layer's norms; they choose their
steps from them at construction."""
fusions = None
if (
@@ -2983,7 +2983,7 @@ class DeepseekV2Model(nn.Module):
)
install_cutedsl_fusion(
# PP pads self.layers with PPMissingLayer, which has no communicator.
# PP pads self.layers with PPMissingLayer, which has no stage boundaries.
# A pipeline exit completes any handoff before sending its tensors.
self.layers[self.start_layer : self.end_layer],
hidden_size=config.hidden_size,
+1 -1
View File
@@ -440,7 +440,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
pre_layernorm=self.post_attention_layernorm if self._use_pre else None,
post_layernorm=self.post_self_attn_layernorm,
)
# The communicator chooses its fused steps from its norms when built.
# The stage boundaries choose their fused steps from their norms when built.
self.attn_boundary, self.ffn_boundary = self._build_stages(
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
+1 -1
View File
@@ -884,7 +884,7 @@ class Glm4MoeDecoderLayer(nn.Module):
)
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
# RMSNorm + FP8 quant into a single kernel in prepare_attn
# RMSNorm + FP8 quant into a single kernel in the attention boundary's read
self.attn_quant_format = ""
self._detect_attn_quant_format()
+1 -1
View File
@@ -319,7 +319,7 @@ class GptOssSparseMoeBlock(nn.Module):
# unpadded slice so the small bf16 router GEMM dimensions stay
# untouched, while the experts call gets to keep the padded view
# and skip the duplicate pad inside the MXFP4 method. The output
# is then trimmed back to the unpadded width so postprocess_layer
# is then trimmed back to the unpadded width so the FFN boundary
# can pair it with the (M, hidden_dim_unpadded) residual.
num_tokens = hidden_states.shape[0]
hidden_dim_unpadded = self.hidden_size
+1 -1
View File
@@ -209,7 +209,7 @@ class KimiDeltaAttention(nn.Module):
safe_gate / lower_bound: clamp the forget gate from below. ``lower_bound``
is ignored unless ``safe_gate`` is set.
reduce_results: forwarded to ``o_proj``; set False when the caller does
its own all-reduce (e.g. a fused MoE/attention communicator).
its own all-reduce (e.g. a fused MoE/attention boundary).
shard_on_attn_tp: shard on the attention-TP group instead of the global
TP group. Required under DP attention, where attn_tp_size < tp_size.
v_head_dim: asymmetric value head dim; defaults to the key head dim.
+1 -1
View File
@@ -683,7 +683,7 @@ class ForwardFlags:
"sp_active": False,
}
# Read/written inside compiled graphs (vocab embedding, communicator,
# Read/written inside compiled graphs (vocab embedding, layer boundaries,
# EP dispatch, DP gather/scatter, MLP/MoE skip-AR): plain-slot backed.
# Before moving a flag out of this set, prove no read/write site sits
# under torch.compile.