[Refactor] Document layer boundary contracts and integration (#41557)
This commit is contained in:
@@ -626,9 +626,10 @@ class ExecComm(msgspec.Struct):
|
||||
help="Select FFN boundary reduction: ar uses all-reduce then token "
|
||||
"redistribution; rs and rsv permit fixed-size and variable-size "
|
||||
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
|
||||
"Unsupported paths fall back to ar. auto resolves the model default "
|
||||
"for all FFN layers. Required attention and MoE collectives and "
|
||||
"all-reduce fusion are unaffected.",
|
||||
"Unsupported paths fall back to ar. The option applies to the FFN "
|
||||
"stages of decoders built with stage boundaries, where auto resolves "
|
||||
"the model default; other models ignore it. Required attention and "
|
||||
"MoE collectives and all-reduce fusion are unaffected.",
|
||||
resolvable=True,
|
||||
),
|
||||
] = "auto"
|
||||
|
||||
@@ -422,7 +422,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# By default a layer communicator reduces the output under DP attention.
|
||||
# By default the stage boundary reduces the output under DP attention.
|
||||
if reduce_results is None:
|
||||
reduce_results = not is_dp_attention_enabled()
|
||||
self.out_proj = RowParallelLinear(
|
||||
|
||||
@@ -11,4 +11,4 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""What the communicator takes from the modules it serves."""
|
||||
"""What the stage boundaries take from the modules they serve."""
|
||||
|
||||
@@ -57,7 +57,7 @@ class AttentionInputs:
|
||||
self.hidden_states_ = None
|
||||
self.qkv_latent_ = None
|
||||
# When True, hidden_states_local is already attn_tp-gathered upstream
|
||||
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
|
||||
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
|
||||
self.is_pre_gathered = is_pre_gathered
|
||||
|
||||
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
|
||||
|
||||
@@ -156,11 +156,19 @@ def make_boundary(
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""Bind only the update capabilities that can reach this edge.
|
||||
"""Bind the consumer half of an edge at construction time.
|
||||
|
||||
Execution receives the producer's actual update from the residual stream.
|
||||
A single capability needs no dispatcher; multiple capabilities select one
|
||||
of the already constructed paths at the consumer.
|
||||
Args:
|
||||
edge: Producer/consumer contracts, residual rows and update capabilities.
|
||||
fusions: Ordered candidates for completing a declared sum with the read.
|
||||
carried_fusions: Ordered candidates accepting dynamically carried work.
|
||||
cp_moves: Context-parallel strategy operations for this edge, if needed.
|
||||
enters_stack: Whether the read must initialize the stack's residual.
|
||||
|
||||
Returns:
|
||||
A Boundary whose prepare accepts the actual update from the stream.
|
||||
Multiple update capabilities select among preconstructed paths; a single
|
||||
capability needs no runtime dispatcher.
|
||||
"""
|
||||
capabilities = edge.update_capabilities
|
||||
if not capabilities:
|
||||
@@ -287,10 +295,18 @@ def _bind_consumer(
|
||||
def make_output_boundary(
|
||||
edge: EdgeDecl, *, cp_moves: Optional[CpMoves] = None
|
||||
) -> Boundary:
|
||||
"""The producer's half of ``edge``, out of a layer's last stage onto the
|
||||
rows the layer hands on (``edge.need``), whose consumer runs in the next
|
||||
layer: the postprocess that moves the output there. ``cp_moves`` for an
|
||||
edge that returns across attention CP."""
|
||||
"""Bind the producer half of a layer/branch handoff.
|
||||
|
||||
Args:
|
||||
edge: Output contract and destination/residual rows. Deferred updates
|
||||
must outlive the producer; pipeline handoffs require a plain add
|
||||
or a residual already written by the producer.
|
||||
cp_moves: Context-parallel return operations, when the edge needs them.
|
||||
|
||||
Returns:
|
||||
A Boundary with fixed transport or a marker for batch-dependent DP
|
||||
transport. The receiver binds its read independently.
|
||||
"""
|
||||
update = edge.produced.update
|
||||
if not getattr(update, "at_producer", False):
|
||||
if not getattr(update, "can_defer_across_layers", False):
|
||||
|
||||
@@ -108,10 +108,15 @@ def _input_can_be_scattered() -> bool:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageEdges:
|
||||
"""One stage's incoming and outgoing declarations for one batch variant.
|
||||
"""One stage's incoming and outgoing contracts for one batch variant.
|
||||
|
||||
No adjacent stage object is required. Edges can come from a local model
|
||||
assembler, pipeline contract or branch adapter.
|
||||
Fields:
|
||||
incoming: Consumer-side contract used to bind prepare.
|
||||
outgoing: Producer-side contract used to bind exit transport.
|
||||
handoff: Optional attention adapter run after input preparation/movement.
|
||||
cp_moves: Strategy-specific context-parallel gather and return operations.
|
||||
|
||||
No executable neighbouring stage or neighbouring norm is required.
|
||||
"""
|
||||
|
||||
incoming: EdgeDecl
|
||||
@@ -125,7 +130,26 @@ def _requires_branch_input(*args, **kwargs):
|
||||
|
||||
|
||||
class StagePlan:
|
||||
"""Precomputed paths for one stage; never owns the neighbouring norm."""
|
||||
"""Bind reusable entry/exit paths from a stage's resolved declarations.
|
||||
|
||||
Args:
|
||||
kind: Boundary role selecting attention or FFN fusion adapters.
|
||||
norm: This stage's consumer normalization module.
|
||||
variants: Mapping from BatchVariant to StageEdges. Construction binds
|
||||
each entry once; a forward selects an existing path by batch facts.
|
||||
enters_stack: Whether this stage initializes the residual from embeddings.
|
||||
prepared_input: Branch reusing an already-read input; requires branch_input
|
||||
instead of a normal prepare to avoid repeating the read/update.
|
||||
terminal: Whether the stage ends the model's layer stack.
|
||||
direct_handoff: Attention can publish its output with finish instead of
|
||||
an exit scope and output transport.
|
||||
qkv_latent_func: Optional hook for prepared attention input.
|
||||
fusions: Optional backend provider of ordered consumer fusion candidates
|
||||
and the producer deferral policies (see make_attn_stage).
|
||||
|
||||
The plan owns static paths, not per-forward tensors or a neighbour's norm.
|
||||
Runtime residual state belongs to the ForwardBatch's ResidualStream.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -27,7 +27,14 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ADD, NORM_READ
|
||||
|
||||
|
||||
class ProducerReduction(Enum):
|
||||
"""Whether compute always leaves its sum or follows the boundary scope."""
|
||||
"""Describe compute's cooperation with its output boundary.
|
||||
|
||||
PARTIAL is attention-only: finish() publishes its partial sum. SCOPED
|
||||
follows exit() flags for an FFN or single-stage mixer. LOCAL_TAIL is FFN-only:
|
||||
compute adds a replicated component after its internal sum, so the
|
||||
boundary never defers that sum to the next layer. A selected reduce-scatter
|
||||
still applies, so compute must add the tail only when it completes the sum.
|
||||
"""
|
||||
|
||||
PARTIAL = auto()
|
||||
SCOPED = auto()
|
||||
@@ -50,8 +57,14 @@ class BatchVariant(Enum):
|
||||
|
||||
|
||||
class StageInput(msgspec.Struct, frozen=True):
|
||||
"""The rows a stage's consumer needs: sharded over the token axes its
|
||||
compute group does not span."""
|
||||
"""Declare the consumer's input rows and read operation.
|
||||
|
||||
Fields:
|
||||
layout: Token sharding required by compute.
|
||||
gathers_itself: Token axes compute can gather internally; the boundary
|
||||
may hand it input still sharded over those axes.
|
||||
read: Operation that derives input from the updated residual.
|
||||
"""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
@@ -62,14 +75,25 @@ class StageInput(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
"""What a stage's producer hands the boundary after it, fixed at
|
||||
construction: its rows, the group its output is summed over, and when it
|
||||
leaves that sum to the boundary instead of completing it.
|
||||
"""Declare producer rows and permitted reduction handoffs at construction.
|
||||
|
||||
What one output actually owes is carried with the value: a plain
|
||||
tensor handed across layers is complete, an ``UnreducedOutput`` names what
|
||||
is left. A raw compute output entering a boundary is read with this
|
||||
declaration and the boundary's decision for the batch."""
|
||||
Fields:
|
||||
layout: Token sharding of the producer contribution.
|
||||
group: Named sum group, or None when there is no reduction.
|
||||
always_leaves: Compute always returns a partial sum.
|
||||
leaves_for_next_layer: Compute can skip reduction under the exit scope
|
||||
and let the following layer complete it.
|
||||
leaves_for_reduce_scatter: Compute can leave reduction to a fixed-size
|
||||
reduce-scatter selected by the boundary.
|
||||
leaves_for_reduce_scatterv: Compute can leave reduction to the selected
|
||||
variable-size attention-DP combine.
|
||||
update: Producer residual operation; None on an arrival contract that
|
||||
declares capabilities and obtains the actual update from the stream.
|
||||
transform: Optional operation on the contribution before residual update.
|
||||
|
||||
These permissions are not evidence that a particular output is partial.
|
||||
The exit decision and ResidualStream record what that output actually owes.
|
||||
"""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
@@ -81,8 +105,7 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
leaves_for_next_layer: bool = False
|
||||
leaves_for_reduce_scatter: bool = False
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
# Whether the selected attention-DP combine may complete this sum.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
# How the producer's output is written into the residual. An arrival
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
@@ -91,17 +114,31 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
"""Resolved input and output contracts for one stage and batch variant.
|
||||
|
||||
Fields:
|
||||
input: Consumer row requirement and read operation.
|
||||
output: Producer row, reduction and residual-update contract.
|
||||
"""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
rows the residual is on before and after the boundary."""
|
||||
"""Describe both sides of a boundary and the residual's row movement.
|
||||
|
||||
Fields:
|
||||
produced: Producer output contract as seen by this side of the edge.
|
||||
need: Consumer input contract.
|
||||
residual: Residual layout before the boundary.
|
||||
residual_to: Residual layout after the boundary.
|
||||
residual_joins_sum: Whether one rank may add the residual into a partial
|
||||
before reduction; valid only for an eligible plain-add update.
|
||||
update_capabilities: Allowed values of StageUpdate.adds_plainly for
|
||||
arriving contributions. Empty means use produced.update's capability.
|
||||
The actual update object travels with the residual stream.
|
||||
"""
|
||||
|
||||
produced: StageOutput
|
||||
need: StageInput
|
||||
@@ -145,7 +182,27 @@ class CpMoves(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class StageEntry(msgspec.Struct, frozen=True):
|
||||
"""The boundary into one of a layer's stages, as the layer runs it."""
|
||||
"""Bound consumer operations for one batch variant.
|
||||
|
||||
Fields:
|
||||
prepare: Callable(hidden_states, residual, forward_batch, norm, **call)
|
||||
returning compute input and updated residual; finishes owed work,
|
||||
applies the producer update and performs the consumer read.
|
||||
input_rows: Layout handed to compute after preparation and input_move.
|
||||
input_move: Optional movement after prepare, before the compute handoff.
|
||||
handoff: Optional callable(input, forward_batch, qkv_latent_func) that
|
||||
adapts already-placed input for attention.
|
||||
capture_move: Optional movement of the updated residual back onto the
|
||||
producer's rows for auxiliary capture.
|
||||
capture_move_allocates: Whether capture_move returns fresh storage (a
|
||||
gather) rather than a view (a cut); capture then retains it
|
||||
without copying.
|
||||
input_sum: Statically owed sum group for an otherwise raw input tensor.
|
||||
preserves_residual: Optional (residual, batch) predicate certifying this
|
||||
input path leaves residual untouched, including backend fallback.
|
||||
capture_preserves_residual: Same guarantee for the local following stage,
|
||||
copied at construction so auxiliary capture can retain its input.
|
||||
"""
|
||||
|
||||
# Completes what the input owes, writes the previous stage's output into
|
||||
# the residual and reads this stage's input:
|
||||
@@ -169,7 +226,16 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class StageSteps(msgspec.Struct, frozen=True):
|
||||
"""One batch variant of one stage: its input and output boundary paths."""
|
||||
"""Precomputed entry and exit work for one stage and batch variant.
|
||||
|
||||
Fields:
|
||||
entry: Bound consumer preparation and handoff.
|
||||
output: Producer contract used by the exit decision.
|
||||
output_move: Fixed output transport, or None when absent or chosen per
|
||||
batch by the attention-DP exit path.
|
||||
output_move_completes_sum: Whether that move also reduces the output.
|
||||
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
|
||||
"""
|
||||
|
||||
entry: StageEntry
|
||||
output: StageOutput
|
||||
@@ -181,9 +247,11 @@ class StageSteps(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
"""Which of a decoder layer's two stages a layer that is one stage takes
|
||||
the place of: its norm, its read and update, its fused kernels and its
|
||||
entry in the layer's steps."""
|
||||
"""Select attention/mixer or FFN input/output adapters.
|
||||
|
||||
This is a boundary role, not a compute implementation or a requirement
|
||||
that layers contain exactly two alternating stages.
|
||||
"""
|
||||
|
||||
ATTENTION = auto()
|
||||
FFN = auto()
|
||||
|
||||
@@ -344,7 +344,9 @@ class OutputBoundary:
|
||||
|
||||
|
||||
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
|
||||
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
|
||||
"""Admit ordinary single-sum outputs, plus LoRA and TP1 shared-expert
|
||||
outputs when a fused consumer (the backend's can_defer_all_reduce,
|
||||
FlashInfer or aiter) can take them.
|
||||
|
||||
A fused-kernel fallback completes the partial-output contract selected
|
||||
before the producer ran, without recomputing the producer's LoRA path.
|
||||
@@ -379,9 +381,18 @@ def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool
|
||||
|
||||
|
||||
class FfnCompletion(msgspec.Struct, frozen=True):
|
||||
"""One FFN's reduction decision. The flags are published while the FFN runs;
|
||||
``complete(hidden_states, residual)`` then either wraps the output for the
|
||||
next layer's input to complete, or runs this layer's postprocess step."""
|
||||
"""One decision shared by compute flags and the matching output completion.
|
||||
|
||||
Fields:
|
||||
defer_moe_finalize: Compute may return a producer-specific finalize handoff.
|
||||
fuse_mlp_allreduce: Compute skips its all-reduce; completion runs it later
|
||||
or carries it to the next consumer, whether fused there or unfused.
|
||||
mlp_reduce_scatter: Compute leaves reduction to the selected scatter path.
|
||||
complete: Callable(output, residual) returning the completed or wrapped
|
||||
output and its corresponding residual rows.
|
||||
|
||||
Do not independently reselect completion after compute has used these flags.
|
||||
"""
|
||||
|
||||
defer_moe_finalize: bool
|
||||
fuse_mlp_allreduce: bool
|
||||
@@ -402,8 +413,10 @@ class MixerExit:
|
||||
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
|
||||
row-parallel output projection to skip the all-reduce. It skips when the
|
||||
stage's output always leaves its sum (to an FFN stage, which completes it in
|
||||
its input), and when its output declaration permits deferring the sum. The
|
||||
consumer decides whether to fuse its completion with the input norm."""
|
||||
its input), or when its declaration permits deferring the sum to the next
|
||||
attention stage and the batch allows it: TP > 1, no attention DP, and no
|
||||
input-scattered or MoE-CP all-gather layout. The consumer decides whether
|
||||
to fuse its completion with the input norm."""
|
||||
|
||||
__slots__ = (
|
||||
"skips_reduction",
|
||||
@@ -454,9 +467,22 @@ class MixerExit:
|
||||
|
||||
|
||||
class FfnExit:
|
||||
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
|
||||
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
|
||||
``defer_moe_finalize`` on ``get_forward()``."""
|
||||
"""Scope a selected FFN output decision and retain its completion action.
|
||||
|
||||
Args:
|
||||
boundary: OutputBoundary that owns the producer's bound paths.
|
||||
forward_batch: Batch selecting reduction/finalize eligibility.
|
||||
stream: This invocation's residual stream, updated by finish().
|
||||
|
||||
Fields:
|
||||
boundary: The owning output boundary.
|
||||
defer_moe_finalize: Whether compute may return a finalize handoff.
|
||||
fuse_mlp_allreduce: Whether compute leaves its all-reduce to completion.
|
||||
mlp_reduce_scatter: Whether compute leaves reduction to a scatter path.
|
||||
|
||||
The three flags are published on get_forward() only inside the context.
|
||||
finish(output) uses the same selected action after successful compute.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"boundary",
|
||||
|
||||
@@ -186,11 +186,32 @@ def _ffn_decl(
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageDeclaration:
|
||||
"""A computation and its input source, without an execution plan.
|
||||
"""Describe one computation's boundaries without binding kernels or norms.
|
||||
|
||||
``previous`` consumes a producer's output. ``prepared_from`` reuses an
|
||||
already-read input for a branch; the two sources are mutually exclusive.
|
||||
Sources may be reconstructed locally, including across pipeline ranks.
|
||||
Fields:
|
||||
kind: Selects attention/mixer or FFN boundary adapters; does not run
|
||||
the corresponding computation.
|
||||
read: Consumer operation that derives compute input from the residual.
|
||||
update: Producer operation that writes compute output into the residual.
|
||||
sparse: Whether this FFN is a MoE; used to resolve input/output rows.
|
||||
Expert routing and all-to-all remain inside the MoE computation.
|
||||
terminal: Whether this stage ends the model's layer stack. Prevents
|
||||
leaving work that requires a following layer; a finalize handoff
|
||||
may still reach the terminal norm when the fusion provider allows it.
|
||||
output_transform: Optional operation on the FFN contribution before
|
||||
residual update, with an explicit reduction-order contract.
|
||||
reduction: Whether compute always leaves a partial sum, obeys the
|
||||
exit scope, or adds a replicated component after its own sum.
|
||||
gathers_tp_input: Whether attention gathers TP-sharded input itself.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
handoff_rows: Required FFN output rows at the layer or branch handoff.
|
||||
previous: Declaration whose output this stage consumes. It may be
|
||||
reconstructed locally, including across pipeline ranks.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
Mutually exclusive with previous; avoids a second update/read.
|
||||
|
||||
Sources contain declarations, never executable boundary objects.
|
||||
"""
|
||||
|
||||
kind: StageKind
|
||||
@@ -217,10 +238,16 @@ class StageDeclaration:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageConnection:
|
||||
"""The producer's exit and consumer's entry for each batch variant.
|
||||
"""Immutable, norm-free contracts for the two sides of a connection.
|
||||
|
||||
Immutable, norm-free declarations. Independently constructed equivalent
|
||||
connections work across pipeline partitions; object identity is irrelevant.
|
||||
Fields:
|
||||
producer: Source declaration, or None at stack entry.
|
||||
consumer: Destination declaration, or None at a layer/stack handoff.
|
||||
exits: Producer-side EdgeDecl for each supported BatchVariant.
|
||||
entries: Consumer-side EdgeDecl for each supported BatchVariant.
|
||||
|
||||
Equivalent declarations can be reconstructed across pipeline partitions;
|
||||
adjacent layers need not share the same connection object.
|
||||
"""
|
||||
|
||||
producer: Optional[StageDeclaration]
|
||||
@@ -243,6 +270,22 @@ def declare_attn(
|
||||
reduction=ProducerReduction.PARTIAL,
|
||||
gathers_tp_input=True,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
Args:
|
||||
previous: Producer declaration whose output this stage consumes.
|
||||
prepared_from: Source of an already-read branch input, instead of previous.
|
||||
read: Input operation; defaults to normalization with quantization support.
|
||||
update: Operation that adds this stage's output to the residual.
|
||||
terminal: Whether this stage ends the model's layer stack.
|
||||
reduction: PARTIAL for an output projection that always skips reduction;
|
||||
SCOPED for a mixer that follows its exit scope's reduction decision.
|
||||
LOCAL_TAIL is rejected for attention stages.
|
||||
gathers_tp_input: Whether compute gathers attention-TP input slices itself.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
"""
|
||||
if reduction is ProducerReduction.LOCAL_TAIL:
|
||||
raise ValueError("LOCAL_TAIL is not supported for attention stages")
|
||||
return StageDeclaration(
|
||||
@@ -271,6 +314,29 @@ def declare_ffn(
|
||||
reduction=ProducerReduction.SCOPED,
|
||||
handoff_rows=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
Args:
|
||||
previous: Producer declaration whose output this stage consumes.
|
||||
prepared_from: Source of an already-read branch input, instead of previous.
|
||||
sparse: Whether the FFN is a MoE; EP dispatch stays inside compute.
|
||||
read: Operation deriving FFN input from the residual.
|
||||
update: Operation writing FFN output into the residual.
|
||||
terminal: Whether this stage ends the model's layer stack.
|
||||
output_transform: Optional contribution transform before residual update.
|
||||
next_sparse: Whether the next decoder layer's FFN is sparse; used only
|
||||
to derive the TBO handoff when handoff_rows is not supplied.
|
||||
dense_tp_size: Dense compute width: None for configuration, 1 for local
|
||||
compute, or the full TP size.
|
||||
reduction: How compute cooperates with the exit's reduction decision:
|
||||
SCOPED (default) follows the exit scope; LOCAL_TAIL marks a
|
||||
replicated tail after the sum. PARTIAL is rejected for FFN stages.
|
||||
handoff_rows: Explicit output-row requirement; otherwise derived from
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
"""
|
||||
if reduction is ProducerReduction.PARTIAL:
|
||||
raise ValueError("PARTIAL is not supported for ffn stages")
|
||||
return StageDeclaration(
|
||||
@@ -522,11 +588,23 @@ def make_attn_stage(
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this attention's boundaries, then bind its norm and hooks.
|
||||
"""Resolve one attention/mixer's boundaries and bind its input norm.
|
||||
|
||||
``following`` supplies the consumer declaration for a stage inside a
|
||||
layer. With no following stage, the output is handed across a layer or
|
||||
stack boundary; the receiver binds its own read independently.
|
||||
Args:
|
||||
declaration: Attention StageDeclaration, including its input source.
|
||||
norm: This consumer's normalization module, never its neighbour's norm.
|
||||
following: Local consumer declaration whose previous is declaration.
|
||||
None leaves a layer/stack handoff for an independently bound reader.
|
||||
qkv_latent_func: Optional attention input hook, invoked after preparation
|
||||
and movement onto the compute input rows.
|
||||
fusions: Optional backend provider. Consumer side: ordered
|
||||
attention_input(plan) and ffn_input(plan) candidates. Producer side
|
||||
(FFN exit): can_defer_finalize(plan, batch), called on every exit,
|
||||
and can_defer_all_reduce(plan, batch), called when LoRA or TP1 shared
|
||||
experts are enabled.
|
||||
|
||||
Returns:
|
||||
A StageBoundary with precomputed paths for supported batch variants.
|
||||
"""
|
||||
if declaration.kind is not StageKind.ATTENTION:
|
||||
raise TypeError("make_attn_stage requires an attention declaration")
|
||||
@@ -548,7 +626,18 @@ def make_ffn_stage(
|
||||
following: Optional[StageDeclaration] = None,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
|
||||
"""Resolve one FFN's boundaries and bind its input norm.
|
||||
|
||||
Args:
|
||||
declaration: FFN StageDeclaration, including its input source.
|
||||
norm: This FFN's input normalization module.
|
||||
following: Local consumer declaration whose previous is declaration;
|
||||
None leaves a layer/stack handoff for an independently bound reader.
|
||||
fusions: Optional backend fusion provider, as in make_attn_stage.
|
||||
|
||||
Returns:
|
||||
A StageBoundary. Expert routing and all-to-all stay inside compute.
|
||||
"""
|
||||
if declaration.kind is not StageKind.FFN:
|
||||
raise TypeError("make_ffn_stage requires an FFN declaration")
|
||||
incoming, outgoing = _connections(declaration, following)
|
||||
@@ -564,18 +653,23 @@ def make_ffn_stage(
|
||||
def make_stages(
|
||||
*stages, previous=None, prepared_from=None, following=None, terminal=False
|
||||
):
|
||||
"""Bind a local linear sequence and return its independent boundaries.
|
||||
"""Bind a local linear sequence of any positive number of stages.
|
||||
|
||||
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
|
||||
Options are the stage-specific keyword arguments of ``make_attn_stage``
|
||||
or ``make_ffn_stage``; they are consumed only during construction.
|
||||
Declarations in the sequence have no input source: this function connects
|
||||
them in order. Only the final boundary is terminal when requested.
|
||||
Args:
|
||||
*stages: Items of (declaration, norm) or (declaration, norm, options).
|
||||
Declarations must have no source or terminal flag. Options are
|
||||
constructor keywords: fusions, and qkv_latent_func for attention.
|
||||
previous: External producer declaration consumed by the first stage.
|
||||
prepared_from: Already-read input declaration reused by the first stage
|
||||
of a branch; mutually exclusive with previous.
|
||||
following: External consumer declaration after the last local stage.
|
||||
None denotes a layer/stack handoff with an independently bound read.
|
||||
terminal: Marks only the final stage as the end of the model's stack.
|
||||
|
||||
``previous`` describes an external producer's output. ``prepared_from``
|
||||
describes an already-read branch input, obtainable from the source
|
||||
boundary's immutable ``declaration``. Neither takes an executable boundary.
|
||||
No sequence object or runtime routing is retained.
|
||||
Returns:
|
||||
A tuple of independent StageBoundary objects in declaration order.
|
||||
Sources are connected on copied declarations; caller inputs are unchanged.
|
||||
No sequence object or runtime routing is retained.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("make_stages needs at least one stage")
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""FlashInfer MNNVL CuTe DSL AllReduce fusion, shared across architectures.
|
||||
|
||||
Two patterns share one workspace, both consumed at the next layer's input
|
||||
RMSNorm: AR + residual + RMSNorm, and the same with the MoE finalize and the
|
||||
shared-expert add folded in when the runner hands back a MoeFinalizeHandoff.
|
||||
RMSNorm (and, with terminal_finalize, at the final norm): AR + residual +
|
||||
RMSNorm, and the same with the MoE finalize and the shared-expert add folded in
|
||||
when the runner hands back a MoeFinalizeHandoff.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -201,7 +202,10 @@ class CuteDSLFusion:
|
||||
At the FFN input: the attention output's AR + add + norm. At the FFN exit:
|
||||
handing off a deferred MoE finalize with its unfused completion.
|
||||
install_cutedsl_fusion supplies the service and producer capability;
|
||||
consumer eligibility is checked only where the output is consumed."""
|
||||
consumer eligibility is checked only where the output is consumed. Producer
|
||||
policy: can_defer_all_reduce keeps a LoRA/TP1 shared-expert sum deferrable
|
||||
while the workspace can take it; can_defer_finalize also covers a terminal
|
||||
FFN when terminal_finalize is installed."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.service: CuteDSLFusionService | None = None
|
||||
@@ -375,7 +379,10 @@ def install_cutedsl_fusion(
|
||||
) -> CuteDSLFusionService | None:
|
||||
"""One shared workspace handle per fusion-enabled layer, or None.
|
||||
|
||||
Every entry of ``layers`` carries its attention and FFN stages.
|
||||
Every entry of ``layers`` carries ``attn_boundary`` and ``ffn_boundary``.
|
||||
terminal_finalize lets the last layer hand its finalize to the final norm;
|
||||
the model must then pass the returned service as
|
||||
residual_batch.norm(handoff_norm=...).
|
||||
"""
|
||||
if get_flags().moe.in_speculative_scope:
|
||||
# A draft shares the target's process, which holds one workspace.
|
||||
|
||||
@@ -48,8 +48,16 @@ class TokenAxis(Enum):
|
||||
|
||||
|
||||
class Layout(msgspec.Struct, frozen=True):
|
||||
"""The token axes a rank's rows are sharded over. Single-rank axes are left
|
||||
out, so two layouts are equal exactly when every rank holds the same tokens."""
|
||||
"""Describe token-row sharding under the active parallel topology.
|
||||
|
||||
Fields:
|
||||
sharded: Axes across which ranks own different token rows. Factories
|
||||
omit size-one axes through sharded_over().
|
||||
|
||||
This describes supported token partitions, not a general device mesh or
|
||||
hidden-dimension placement. Row order and padding come from the batch's
|
||||
shared token mapping, not from per-tensor metadata in this object.
|
||||
"""
|
||||
|
||||
sharded: FrozenSet[TokenAxis]
|
||||
|
||||
|
||||
@@ -399,7 +399,7 @@ def scatter_moe_cp_output(
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
||||
"""Return a MoE output computed on the MoE-CP-gathered rows to this rank's attention rows.
|
||||
|
||||
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
||||
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
||||
|
||||
@@ -22,11 +22,15 @@ from sglang.srt.distributed import GroupCoordinator
|
||||
|
||||
|
||||
class OutputTransform(msgspec.Struct, frozen=True):
|
||||
"""An operation on the producer contribution, before residual addition.
|
||||
"""Transform a producer contribution before its residual update.
|
||||
|
||||
Some existing implementations apply it to a partial before reduce-scatter
|
||||
but to a complete value after all-reduce. These are explicit alternatives,
|
||||
not an assumption that an arbitrary operation commutes with reduction.
|
||||
Fields:
|
||||
apply: Callable(tensor) returning the transformed contribution.
|
||||
before_reduce_scatter: Permit this operation on a partial before
|
||||
reduce-scatter. Otherwise the transform follows completion.
|
||||
|
||||
This explicitly preserves an implementation's order; it does not imply
|
||||
that arbitrary transforms commute with reduction or floating-point rounding.
|
||||
"""
|
||||
|
||||
apply: Callable[[torch.Tensor], torch.Tensor]
|
||||
@@ -34,10 +38,18 @@ class OutputTransform(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its sum, left for the next layer's input.
|
||||
Hand it to the next layer, or pass it through reduce_output() before reading
|
||||
it any other way. The producer says what is owed: one all-reduce over
|
||||
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
|
||||
"""Internal adapter value describing an unfinished reduction.
|
||||
|
||||
Fields:
|
||||
partial: Tensor containing this rank's contribution to the sum.
|
||||
group: All-reduce group when no redistribution callable is supplied.
|
||||
reduce_and_redistribute: Callable(partial) completing the sum and moving
|
||||
it to destination rows; takes precedence over group.
|
||||
|
||||
Exits hand this form to ResidualStream.leave(), which exposes an opaque
|
||||
OwedOutput to models. Low-level adapters use reduce_output() before reading
|
||||
it. A group is required when reduce_and_redistribute is absent.
|
||||
"""
|
||||
|
||||
partial: torch.Tensor
|
||||
group: Optional[GroupCoordinator] = None
|
||||
@@ -48,7 +60,8 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
|
||||
class HandoffOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes work only its producer knows how to do (a
|
||||
MoE's finalize and sum), left for the next layer's input. A fused kernel
|
||||
MoE's finalize and sum), left for the next layer's input or for a terminal
|
||||
norm that accepts it (residual_batch.norm(handoff_norm=...)). A fused kernel
|
||||
there may do that work together with its own; anything else passes it
|
||||
through reduce_output(), which calls ``complete()``."""
|
||||
|
||||
|
||||
@@ -436,8 +436,8 @@ def _mlp_input_gather_moe_cp(
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
||||
TP_ATTN_FULL."""
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays on
|
||||
this rank's attention rows."""
|
||||
# Early return on empty tensor is safe for MOE_CP because:
|
||||
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
||||
# so no rank hits this path while others proceed to the allgather.
|
||||
|
||||
@@ -22,7 +22,19 @@ import torch
|
||||
|
||||
|
||||
class StageUpdate(Protocol):
|
||||
"""How a stage's output is written into the residual."""
|
||||
"""Producer-owned operation writing a contribution into the residual.
|
||||
|
||||
Fields:
|
||||
adds_plainly: Whether update is plain addition, permitting compatible
|
||||
add+norm fusion and adding the residual on one rank before a sum.
|
||||
at_producer: Whether the producer writes the residual at its exit rather
|
||||
than leaving the operation for the next prepare.
|
||||
can_defer_across_layers: Whether parameters/state remain valid after
|
||||
leaving the producer layer, including any offload or state reuse.
|
||||
|
||||
The shard conversion methods must move any update-associated state together
|
||||
with the residual. Nonlinear updates must not claim adds_plainly.
|
||||
"""
|
||||
|
||||
# A plain add, which one rank may run before the sum it adds into
|
||||
# completes (the DP partial order, a residual that joins the attention
|
||||
@@ -36,7 +48,12 @@ class StageUpdate(Protocol):
|
||||
can_defer_across_layers: bool
|
||||
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the output written into it."""
|
||||
"""Write hidden_states, the producer contribution, into residual.
|
||||
|
||||
Implementations define their residual representation and return the updated
|
||||
tensor. Boundary ordering may put a plain add before an eligible sum.
|
||||
This operation does not perform the consumer's normalization.
|
||||
"""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual, with whatever the
|
||||
@@ -47,7 +64,18 @@ class StageUpdate(Protocol):
|
||||
|
||||
|
||||
class StageRead(Protocol):
|
||||
"""How a stage reads its input from the residual."""
|
||||
"""Consumer-owned operation deriving compute input from the residual.
|
||||
|
||||
Fields:
|
||||
norms_plainly: Read is normalization/optional quantization without changing
|
||||
the residual, allowing compatible fused add+norm implementations.
|
||||
before_gather: Preserve this read on source rows before a DP gather.
|
||||
|
||||
enter initializes the stack residual. read consumes an already-written
|
||||
residual; update_and_read first applies the actual producer's update.
|
||||
Both reads return (compute_input, residual), preserving their kernel's
|
||||
rounding order rather than normalizing a separately rounded snapshot.
|
||||
"""
|
||||
|
||||
# The input is the residual's norm, in the quantization the call asks for,
|
||||
# and the residual is left as it is: what a fused add + norm kernel computes,
|
||||
@@ -66,8 +94,19 @@ class StageRead(Protocol):
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple:
|
||||
"""The input and the residual, from a residual that already holds the
|
||||
previous stage's output."""
|
||||
"""Read input from a residual that already includes the producer update.
|
||||
|
||||
Args:
|
||||
residual: Updated residual tensor in this implementation's representation.
|
||||
norm: Consumer normalization module.
|
||||
quant_format: Requested input quantization format, or empty for the
|
||||
read's default format; supported values belong to the implementation.
|
||||
post_residual_addition: Kept for signature parity with update_and_read;
|
||||
current read implementations ignore or reject it.
|
||||
|
||||
Returns:
|
||||
(compute_input, residual), potentially with a quantized input format.
|
||||
"""
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
@@ -78,8 +117,20 @@ class StageRead(Protocol):
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple:
|
||||
"""Write the previous stage's output into the residual with its
|
||||
producer's ``update``, then read the input."""
|
||||
"""Apply a producer update and read input while preserving kernel ordering.
|
||||
|
||||
Args:
|
||||
update: Actual producer's StageUpdate, not a consumer-inferred operation.
|
||||
hidden_states: Producer contribution after required communication.
|
||||
residual: Previous residual, or None at entry where supported.
|
||||
norm: Consumer normalization module.
|
||||
quant_format: Requested input quantization format, as in read().
|
||||
post_residual_addition: Optional extra added after update, before norm.
|
||||
|
||||
Returns:
|
||||
(compute_input, residual). A fused implementation can preserve FP32
|
||||
accumulation through norm without materializing a rounded intermediate.
|
||||
"""
|
||||
|
||||
|
||||
class LayerResidual(NamedTuple):
|
||||
|
||||
@@ -85,11 +85,13 @@ def finish_layer_stack(
|
||||
final_norm_takes_handoff: bool = False,
|
||||
preserve_declared: bool = False,
|
||||
) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers. A final norm
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
"""Complete what this layer left for a next layer, for callers that carry
|
||||
an explicit residual tensor or merge TBO microbatches. Stage-boundary
|
||||
decoders use residual_batch.norm(), to_pp() or take_output() instead:
|
||||
to_pp() keeps a declared partial sum for the receiving from_pp(), which
|
||||
this helper would complete early. A final norm that does a producer's
|
||||
handoff together with its own work (``final_norm_takes_handoff``)
|
||||
receives it as it is."""
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.finish(
|
||||
hidden_states,
|
||||
|
||||
@@ -147,7 +147,7 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
|
||||
and _is_flashinfer_available
|
||||
and not is_dp_attention_enabled()
|
||||
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
||||
# cutedsl runs its own fused path from the fusion communicator.
|
||||
# cutedsl runs its own fused path from its stage fusion (fusions/cutedsl.py).
|
||||
and not uses_cutedsl_ar_fusion()
|
||||
and not is_flashinfer_allreduce_unavailable()
|
||||
# Symbolic size checks stay last: under Dynamo tracing they guard on
|
||||
|
||||
@@ -19,6 +19,11 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
|
||||
def start(forward_batch):
|
||||
"""Reset forward_batch.residual_stream at entry to a fresh layer-stack call.
|
||||
|
||||
Call once before the first local stage on an embedding path, not per layer.
|
||||
Pipeline reception and TBO adapters reconstruct their own streams.
|
||||
"""
|
||||
forward_batch.residual_stream = ResidualStream()
|
||||
|
||||
|
||||
@@ -54,6 +59,26 @@ def norm(
|
||||
skip_empty=False,
|
||||
**read_kwargs,
|
||||
):
|
||||
"""Complete the layer stack and apply its terminal normalization.
|
||||
|
||||
Args:
|
||||
hidden_states: Current stream output or opaque owed handle.
|
||||
forward_batch: Batch owning the residual stream.
|
||||
layernorm: Final norm supporting the model's output/residual pair.
|
||||
capture_output: Optional callback retaining the same updated residual;
|
||||
it must copy borrowed storage when retention requires ownership.
|
||||
handoff_norm: Optional adapter with finalize(handoff, residual, gamma)
|
||||
for a producer-specific finalize handoff; when a handoff arrives it
|
||||
cannot be combined with capture. Without it, a handoff is completed
|
||||
unfused first. Its gamma is layernorm.gemma_weight, so the final
|
||||
norm must then be a GemmaRMSNorm.
|
||||
skip_empty: Skip the ordinary norm on a zero-row completed output.
|
||||
**read_kwargs: Extra arguments forwarded to the two-input final norm.
|
||||
|
||||
Returns:
|
||||
Normalized tensor. Add and norm remain together to preserve the kernel's
|
||||
accumulation/rounding order; a snapshot is not used as the norm input.
|
||||
"""
|
||||
hidden_states, residual = current(forward_batch).finish(
|
||||
hidden_states, takes_handoff=handoff_norm is not None
|
||||
)
|
||||
@@ -81,6 +106,17 @@ def norm(
|
||||
|
||||
|
||||
def to_pp(hidden_states, forward_batch, *, preserve_declared=True):
|
||||
"""Export hidden_states and residual as PPProxyTensors.
|
||||
|
||||
Args:
|
||||
hidden_states: Current stream output or opaque owed handle.
|
||||
forward_batch: Batch owning the stream.
|
||||
preserve_declared: True (the default) sends a statically declared
|
||||
partial sum unreduced for the receiver's from_pp to complete; pass
|
||||
False only when the receiver does not declare that sum.
|
||||
|
||||
Runtime-selected completion work is finished before transport.
|
||||
"""
|
||||
hidden_states, residual = current(forward_batch).finish(
|
||||
hidden_states, preserve_declared=preserve_declared
|
||||
)
|
||||
|
||||
@@ -25,7 +25,13 @@ from sglang.srt.layers.layer_boundary.residual import StageUpdate
|
||||
|
||||
|
||||
class CarriedSum(msgspec.Struct, frozen=True):
|
||||
"""Work left by the producer, including a move back to local token rows."""
|
||||
"""Runtime reduction left by an exit decision.
|
||||
|
||||
Fields:
|
||||
group: All-reduce group when no redistribution callable is supplied.
|
||||
reduce_and_redistribute: Optional callable(value) returning a reduced
|
||||
value on destination rows; takes precedence over group.
|
||||
"""
|
||||
|
||||
group: Optional[GroupCoordinator] = None
|
||||
reduce_and_redistribute: Optional[Callable] = None
|
||||
@@ -46,7 +52,15 @@ class DeclaredSum(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class Contribution(msgspec.Struct):
|
||||
"""The sole owner of an output's value, update and remaining work."""
|
||||
"""Own a producer's output, residual update and outstanding completion.
|
||||
|
||||
Fields:
|
||||
value: Contribution tensor, or None for an opaque finalize handoff.
|
||||
update: Producer operation to apply before the consumer reads input.
|
||||
owed: Runtime sum, declared sum, producer-specific finalize, or None.
|
||||
|
||||
Completing owed work clears owed but does not apply the residual update.
|
||||
"""
|
||||
|
||||
value: Optional[torch.Tensor]
|
||||
update: StageUpdate
|
||||
@@ -80,11 +94,20 @@ class OwedOutput(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class ResidualStream:
|
||||
"""Uninitialized, written, or holding one pending producer contribution.
|
||||
"""Own residual state for one forward or TBO microbatch.
|
||||
|
||||
Each forward or TBO microbatch owns its stream. A prepare consumes its
|
||||
pending contribution once; completing a sum for capture leaves the update
|
||||
pending. No layout or batch selection is stored on the value.
|
||||
Args:
|
||||
residual: Already-written residual, or None before stack entry.
|
||||
|
||||
Fields:
|
||||
residual: Residual tensor retained across stages, or None initially.
|
||||
pending: Contribution awaiting a residual update, or None. Its owed
|
||||
field separately records incomplete reduction/finalize work.
|
||||
|
||||
The states are initial (both None), written (residual only), and pending.
|
||||
Prepare consumes pending once and writes the resulting residual. Completing
|
||||
communication for capture leaves the update pending. Layouts and batch-path
|
||||
selection live on the bound stage, not on tensor values.
|
||||
"""
|
||||
|
||||
__slots__ = ("residual", "pending")
|
||||
@@ -95,7 +118,18 @@ class ResidualStream:
|
||||
|
||||
@classmethod
|
||||
def arrive(cls, hidden, residual, update, *, declared_sum=None):
|
||||
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
|
||||
"""Reconstruct a stream from a tensor handoff, including a TBO microbatch.
|
||||
|
||||
Args:
|
||||
hidden: Received output or already-written residual tensor.
|
||||
residual: Separate residual tensor. None means hidden already holds
|
||||
the written residual, rather than an uninitialized stack.
|
||||
update: Producer operation for a separate contribution/residual pair.
|
||||
declared_sum: Static sum owed by hidden when residual is separate.
|
||||
|
||||
Returns:
|
||||
(handle, stream). Use start() for a fresh, uninitialized stack instead.
|
||||
"""
|
||||
stream = cls(residual)
|
||||
if residual is None:
|
||||
return stream.write(hidden), stream
|
||||
@@ -107,6 +141,18 @@ class ResidualStream:
|
||||
return residual
|
||||
|
||||
def leave(self, output, update, *, declared_sum=None):
|
||||
"""Record one producer contribution without applying its residual update.
|
||||
|
||||
Args:
|
||||
output: Complete tensor, UnreducedOutput, or producer-specific HandoffOutput.
|
||||
update: Producer operation that the next boundary must apply.
|
||||
declared_sum: SumGroup statically owed by a raw tensor, or None. Cannot
|
||||
be combined with another output wrapper's completion contract.
|
||||
|
||||
Returns:
|
||||
The tensor when complete, otherwise an opaque OwedOutput. Replacing an
|
||||
unconsumed contribution is an error; retain the returned handle.
|
||||
"""
|
||||
if self.pending is not None:
|
||||
raise RuntimeError("cannot replace an unconsumed producer contribution")
|
||||
if declared_sum is not None:
|
||||
@@ -174,7 +220,19 @@ class ResidualStream:
|
||||
return value.clone() if self.residual is None else value + self.residual
|
||||
|
||||
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
|
||||
"""Export the contribution/residual pair; keep the stream available to readers."""
|
||||
"""Export the output/residual pair without closing or consuming the stream.
|
||||
|
||||
Args:
|
||||
hidden: Current stream tensor or opaque owed handle.
|
||||
takes_handoff: Pass a producer-specific finalize handoff through for a
|
||||
terminal adapter that can consume it; otherwise complete it here.
|
||||
preserve_declared: Export a raw declared partial sum for a receiver
|
||||
whose incoming contract reconstructs that sum (for example PP).
|
||||
|
||||
Returns:
|
||||
(output, residual), completing outstanding work unless explicitly
|
||||
preserved. Does not apply the pending residual update.
|
||||
"""
|
||||
self.check(hidden)
|
||||
if self.pending is None:
|
||||
return hidden, None
|
||||
|
||||
@@ -46,8 +46,16 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class StageBoundary:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
"""Model-facing preparation and completion for one compute stage.
|
||||
|
||||
Args:
|
||||
plan: Precomputed entry/exit paths, this stage's norm and input hooks.
|
||||
declaration: Immutable stage description used to construct related
|
||||
boundaries without exposing this object's execution plan.
|
||||
|
||||
The boundary does not run attention or FFN computation. Per-forward residual
|
||||
state is read from forward_batch.residual_stream, not stored here.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -127,6 +135,24 @@ class StageBoundary:
|
||||
return self.plan.input_on_attention_tp_slices
|
||||
|
||||
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
|
||||
"""Finish the previous contribution and produce this stage's compute input.
|
||||
|
||||
Args:
|
||||
hidden_states: Tensor or opaque owed-output handle belonging to this
|
||||
forward's residual stream. Do not unwrap or modify a partial sum.
|
||||
forward_batch: Batch owning the stream and selecting the prepared path.
|
||||
cache: FFN-only NPU weight cache prefetched on the FFN input path.
|
||||
**call: Attention-only read/adapter options: quant_format,
|
||||
post_residual_addition, captured_last_layer_outputs and capture_output.
|
||||
FFN stages do not accept these options. A capture callback is
|
||||
called as capture_output(value, owned=bool): owned=True means
|
||||
value is fresh storage it may keep, and owned=False means it
|
||||
must copy before retaining.
|
||||
|
||||
Returns:
|
||||
Compute input in the selected read's format (possibly quantized).
|
||||
Updates forward_batch.residual_stream with the resulting residual.
|
||||
"""
|
||||
if cache is not None:
|
||||
call["cache"] = cache
|
||||
prepare = (
|
||||
@@ -153,6 +179,17 @@ class StageBoundary:
|
||||
)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
"""Select and scope one FFN or single-stage mixer's output decision.
|
||||
|
||||
Args:
|
||||
forward_batch: Active batch whose stream receives the compute result.
|
||||
|
||||
Returns:
|
||||
A context manager publishing reduction/finalize flags during compute.
|
||||
Call its finish(output) exactly once after successful compute, including
|
||||
when compute is skipped for an empty input. It completes or carries work
|
||||
using the same decision rather than selecting a second path.
|
||||
"""
|
||||
result = (
|
||||
self.plan.output.ffn_exit(forward_batch, stream=current(forward_batch))
|
||||
if self.kind is StageKind.FFN
|
||||
@@ -163,6 +200,18 @@ class StageBoundary:
|
||||
return result
|
||||
|
||||
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
|
||||
"""Restore the residual stream from a pipeline handoff.
|
||||
|
||||
Args:
|
||||
tensors: PPProxyTensors containing hidden_states and, ordinarily, residual.
|
||||
forward_batch: Batch receiving a new stream.
|
||||
allow_missing_residual: Accept a handoff without the residual key for
|
||||
model paths that explicitly support it.
|
||||
|
||||
Returns:
|
||||
Tensor or owed handle for prepare. A producer-written residual and a
|
||||
declared partial sum are reconstructed from the incoming contract.
|
||||
"""
|
||||
hidden_states, residual = from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=(
|
||||
@@ -181,6 +230,18 @@ class StageBoundary:
|
||||
return current(forward_batch).snapshot(hidden_states)
|
||||
|
||||
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
|
||||
"""Capture an output while preserving the caller's reduction timing.
|
||||
|
||||
Args:
|
||||
hidden_states: Current stream output or opaque owed handle.
|
||||
forward_batch: Batch owning that stream.
|
||||
skip_empty: Return no snapshot for a known zero-row output buffer.
|
||||
|
||||
Returns:
|
||||
(updated_handle, snapshot). Dynamic owed work is completed on the main
|
||||
contribution; a declared sum is reduced only on the snapshot copy.
|
||||
Always use updated_handle afterwards. Plain residual updates are required.
|
||||
"""
|
||||
stream = current(forward_batch)
|
||||
if skip_empty:
|
||||
storage = buffer(hidden_states)
|
||||
@@ -191,6 +252,9 @@ class StageBoundary:
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def postprocess(self, hidden_states, forward_batch):
|
||||
"""Move an FFN output that compute already completed outside exit()
|
||||
(the operation-scheduled TBO path) onto the rows the layer hands on;
|
||||
it chooses no reduction step."""
|
||||
return self.plan.output.postprocess_layer(
|
||||
hidden_states, current(forward_batch), forward_batch
|
||||
)
|
||||
|
||||
@@ -302,7 +302,7 @@ class BaseRunner(ABC):
|
||||
# Nothing else resolves the configured backend, so check the platform.
|
||||
resolve_flashinfer_allreduce_fusion_backend()
|
||||
if not mr.is_draft_worker:
|
||||
# A draft installs no fusion communicator.
|
||||
# A draft installs no CuTe DSL fusion.
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
prepare_cutedsl_fusion,
|
||||
)
|
||||
|
||||
@@ -2633,7 +2633,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
qkv_latent_func: Optional[Callable],
|
||||
output=None,
|
||||
):
|
||||
"""The communicator for this layer's norms; it chooses its boundary
|
||||
"""The stage boundaries for this layer's norms; they choose their
|
||||
steps from them at construction."""
|
||||
fusions = None
|
||||
if (
|
||||
@@ -2983,7 +2983,7 @@ class DeepseekV2Model(nn.Module):
|
||||
)
|
||||
|
||||
install_cutedsl_fusion(
|
||||
# PP pads self.layers with PPMissingLayer, which has no communicator.
|
||||
# PP pads self.layers with PPMissingLayer, which has no stage boundaries.
|
||||
# A pipeline exit completes any handoff before sending its tensors.
|
||||
self.layers[self.start_layer : self.end_layer],
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -440,7 +440,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
pre_layernorm=self.post_attention_layernorm if self._use_pre else None,
|
||||
post_layernorm=self.post_self_attn_layernorm,
|
||||
)
|
||||
# The communicator chooses its fused steps from its norms when built.
|
||||
# The stage boundaries choose their fused steps from their norms when built.
|
||||
self.attn_boundary, self.ffn_boundary = self._build_stages(
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
|
||||
@@ -884,7 +884,7 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
|
||||
# RMSNorm + FP8 quant into a single kernel in prepare_attn
|
||||
# RMSNorm + FP8 quant into a single kernel in the attention boundary's read
|
||||
self.attn_quant_format = ""
|
||||
self._detect_attn_quant_format()
|
||||
|
||||
|
||||
@@ -319,7 +319,7 @@ class GptOssSparseMoeBlock(nn.Module):
|
||||
# unpadded slice so the small bf16 router GEMM dimensions stay
|
||||
# untouched, while the experts call gets to keep the padded view
|
||||
# and skip the duplicate pad inside the MXFP4 method. The output
|
||||
# is then trimmed back to the unpadded width so postprocess_layer
|
||||
# is then trimmed back to the unpadded width so the FFN boundary
|
||||
# can pair it with the (M, hidden_dim_unpadded) residual.
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_dim_unpadded = self.hidden_size
|
||||
|
||||
@@ -209,7 +209,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
safe_gate / lower_bound: clamp the forget gate from below. ``lower_bound``
|
||||
is ignored unless ``safe_gate`` is set.
|
||||
reduce_results: forwarded to ``o_proj``; set False when the caller does
|
||||
its own all-reduce (e.g. a fused MoE/attention communicator).
|
||||
its own all-reduce (e.g. a fused MoE/attention boundary).
|
||||
shard_on_attn_tp: shard on the attention-TP group instead of the global
|
||||
TP group. Required under DP attention, where attn_tp_size < tp_size.
|
||||
v_head_dim: asymmetric value head dim; defaults to the key head dim.
|
||||
|
||||
@@ -683,7 +683,7 @@ class ForwardFlags:
|
||||
"sp_active": False,
|
||||
}
|
||||
|
||||
# Read/written inside compiled graphs (vocab embedding, communicator,
|
||||
# Read/written inside compiled graphs (vocab embedding, layer boundaries,
|
||||
# EP dispatch, DP gather/scatter, MLP/MoE skip-AR): plain-slot backed.
|
||||
# Before moving a flag out of this set, prove no read/write site sits
|
||||
# under torch.compile.
|
||||
|
||||
Reference in New Issue
Block a user