From 305ad5503da016b8ee40392aa4c52cb3538da81c Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:27:23 -0700 Subject: [PATCH] [Refactor] Run Kimi K3's fused collectives through stage boundaries (#42898) --- docs/docs/developer_guide/layer_boundary.mdx | 2 +- python/sglang/srt/layers/attn_residual.py | 16 +- .../srt/layers/layer_boundary/__init__.py | 2 + .../layer_boundary/adapters/attention.py | 10 +- .../srt/layers/layer_boundary/boundary.py | 72 ++++- .../srt/layers/layer_boundary/construction.py | 22 +- .../srt/layers/layer_boundary/contracts.py | 31 +- .../srt/layers/layer_boundary/factories.py | 26 +- .../sglang/srt/layers/layer_boundary/ops.py | 33 ++- .../srt/layers/layer_boundary/prepare.py | 59 +++- .../layer_boundary/residual/__init__.py | 7 + .../layer_boundary/residual/attn_bank.py | 103 ++++++- python/sglang/srt/models/kimi_k3.py | 274 ++++++++++++------ .../unit/layer_boundary/test_append_stages.py | 171 +++++++++++ .../test_attn_bank_residual_ops.py | 262 +++++++++++++++++ .../test_kimi_k3_stage_kernels.py | 204 +++++++++++++ 16 files changed, 1169 insertions(+), 125 deletions(-) create mode 100644 test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index a397cdf50d..eb243cbbe6 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -339,7 +339,7 @@ A backend fusion provider is passed per stage as the `fusions` option, `(declara `install_cutedsl_fusion()` and `prepare_cutedsl_fusion()` find providers through the decoder layer's `attn_boundary` and `ffn_boundary` attributes. -Consumer fusion candidates are ordered. A candidate returning `None` must do so before modifying inputs or starting a collective, so fallback remains valid. `FfnInputFusion` declares the sum group it completes with residual add and norm. The consumer only tries compatible candidates for the declared update/read capabilities. +Consumer fusion candidates are ordered. A candidate returning `None` must do so before modifying inputs or starting a collective, so fallback remains valid. `FfnInputFusion` declares the sum group it completes with residual add and norm. A residual read may supply its own candidates as `completing_fusions`: each `ReadoutFusion` completes a sum with the residual add, ahead of the read, and is tried before the built-in collective; one with `reads=True` also does the read. Where a stage reads its input on this rank's attention-TP slice and then gathers it, a read's `gathering_reads` may do both in one kernel. A stage may declare its own implementation of its gathers over attention TP (`declare_attn(attn_tp_gather=...)`, `declare_ffn(attn_tp_gather=...)`), tried on every batch before the built-in gather. The consumer only tries compatible candidates for the declared update/read capabilities. An `OutputTransform` declares contribution processing before residual update. Its explicit `before_reduce_scatter` setting preserves a supported implementation's order; it is not permission to commute arbitrary operations across a reduction. diff --git a/python/sglang/srt/layers/attn_residual.py b/python/sglang/srt/layers/attn_residual.py index e31114719d..afad3e58b6 100644 --- a/python/sglang/srt/layers/attn_residual.py +++ b/python/sglang/srt/layers/attn_residual.py @@ -460,6 +460,9 @@ class AttnResidual: (num_tokens, block_num, hidden_size) ) self.num_valid_blocks = 0 + # Whether a write snapshotted only this rank's slice of the rows, which + # leaves the bank's other rows stale on this rank. + self.written_on_slice = False if block_residual is not None: # inherited from the previous PP rank self.num_valid_blocks = block_residual.size(1) self.block_residual[:, : self.num_valid_blocks, :].copy_(block_residual) @@ -472,7 +475,11 @@ class AttnResidual: """ bank = self.block_residual if rows is None else self.block_residual[rows] bank[:, self.num_valid_blocks, :].copy_(prefix_sum) + self._count_write(rows) + + def _count_write(self, rows: Optional[slice]) -> None: self.num_valid_blocks += 1 + self.written_on_slice |= rows is not None def forward( self, @@ -488,6 +495,11 @@ class AttnResidual: (the second return value) into the next bank row — fused into the fast kernel (the row streams through its score pass anyway), a standalone .write() copy on every other path.""" + if rows is None and self.written_on_slice: + raise RuntimeError( + "a read of every row after a write of this rank's slice of " + "them would aggregate stale snapshots" + ) nvb = self.num_valid_blocks # Layer 0 attention side: nothing banked yet if nvb == 0: @@ -533,7 +545,7 @@ class AttnResidual: write_bank_row=fused_write, ) if fused_write: - self.num_valid_blocks += 1 # row nvb written in-kernel + self._count_write(rows) # row nvb written in-kernel elif write: self.write(prefix, rows) return normed, prefix @@ -572,7 +584,7 @@ class AttnResidual: if normed is None: return None if write: - self.num_valid_blocks += 1 + self._count_write(rows) return normed, prefix def forward_sp_reduce_scatter( diff --git a/python/sglang/srt/layers/layer_boundary/__init__.py b/python/sglang/srt/layers/layer_boundary/__init__.py index c2b6b3e74c..dbfea55b1c 100644 --- a/python/sglang/srt/layers/layer_boundary/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/__init__.py @@ -34,6 +34,7 @@ from sglang.srt.layers.layer_boundary.contracts import ( InputContract, OutputContract, ProducerReduction, + ReadoutFusion, StageContract, StageKind, StagePath, @@ -96,6 +97,7 @@ __all__ = [ "ExitDecision", "FfnExit", "FfnInputFusion", + "ReadoutFusion", "DeferredFinalize", "GatedResidualState", "IHCState", diff --git a/python/sglang/srt/layers/layer_boundary/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py index eee81fec68..5496aa58b1 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -159,10 +159,14 @@ def get_attn_tp_context(): return ATTN_TP_CONTEXT -def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor: +def attn_tp_gather(tensor: torch.Tensor, *, owned: bool = False) -> torch.Tensor: + """Gather this rank's attention-TP slice of the rows into all of them: in + the local DP buffer, or in a buffer of its own for an output that + outlives the next use of that buffer (``owned``).""" parallel = get_parallel() - if batches_are_unpadded(): - # MLP sync, which sizes the local DP buffer for a batch, does not run. + if owned or batches_are_unpadded(): + # MLP sync, which sizes the local DP buffer for a batch, does not run + # for an unpadded one. gathered = tensor.new_empty( (tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1]) ) diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index 449ba718f3..fa1afbe5d8 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -187,6 +187,7 @@ def bind_entry( cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, attn_input_adapter: Optional[Callable] = None, + attn_tp_gather: Optional[Callable] = None, ) -> EntryPath: """Bind the consumer half of an edge at construction time. @@ -197,6 +198,8 @@ def bind_entry( cp_moves: Context-parallel strategy operations for this edge, if needed. enters_stack: Whether the read must initialize the stack's residual. attn_input_adapter: Hands an attention its input once it is on its rows. + attn_tp_gather: The stage's implementation of a gather over attention + TP, tried before the default one. Returns: An EntryPath whose prepare accepts the actual update from the stream. @@ -218,6 +221,11 @@ def bind_entry( if edge.produced.update is None: raise ValueError("an arrival must declare its update capabilities") capabilities = (edge.produced.update.is_plain_add,) + gather_input = ( + attn_tp_gather_input + if attn_tp_gather is None + else partial(attn_tp_gather_input, gather=attn_tp_gather) + ) paths = { capability: _bind_entry_path( edge, @@ -227,6 +235,7 @@ def bind_entry( cp_moves=cp_moves, enters_stack=enters_stack, attn_input_adapter=attn_input_adapter, + gather_input=gather_input, ) for capability in capabilities } @@ -254,6 +263,7 @@ def _bind_entry_path( cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, attn_input_adapter: Optional[Callable] = None, + gather_input: Callable = attn_tp_gather_input, ) -> EntryPath: """The consumer's half of ``edge``, chosen from the edge's declarations and the producer's update: what a value carries for a batch (a sum or a handoff @@ -277,6 +287,8 @@ def _bind_entry_path( residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, + gather_input=gather_input, + arrives_written=edge.arrives_written, ) completed_step = None if edge.produced.always_partial: @@ -343,7 +355,12 @@ def _bind_entry_path( ) -def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove: +def bind_exit( + edge: EdgeContract, + *, + cp_moves: Optional[CpMoves] = None, + attn_tp_gather: Optional[Callable] = None, +) -> ExitMove: """Bind the producer half of a layer or branch exit. Args: @@ -351,6 +368,8 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit must outlive the producer; pipeline handoffs require a plain add or a residual already written by the producer. cp_moves: Context-parallel return operations, when the edge needs them. + attn_tp_gather: The stage's implementation of a gather over attention + TP, tried before the default one. Returns: An ExitMove with fixed transport or a marker for batch-dependent DP @@ -373,6 +392,20 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit residual=edge.residual, to=edge.residual_to, cp_moves=cp_moves, + attn_tp_gather=attn_tp_gather, + ) + + +def _read_fusions(read, owes, is_plain_add, *, scatters): + """The kernels a read supplies that complete ``owes`` on the rows this + entry completes it on. They take the residual add themselves, so only a + plain one.""" + if not is_plain_add: + return () + return tuple( + f + for f in getattr(read, "completing_fusions", ()) + if f.completes is owes and f.scatters == scatters ) @@ -387,6 +420,8 @@ def _select_entry_step( residual_joins_sum: bool, cp_moves: Optional[CpMoves], enters_stack: bool, + gather_input: Callable = attn_tp_gather_input, + arrives_written: bool = False, ) -> Tuple[Callable, Optional[Callable]]: """The steps into a stage for a value that is complete or owes the sum its producer always leaves, the fused kernels they try first, and the move @@ -447,14 +482,15 @@ def _select_entry_step( or residual not in (produced.layout, need.layout) ): raise NotImplementedError(f"{produced=} {residual=} {need=}") + if owes is SumGroup.ATTN_TP: + step = partial( + _attn_tp_reduce_scatter_update_read, + read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True), + ) + else: + step = _attn_tp_slice_update_read return ( - partial( - _attn_tp_reduce_scatter_update_read - if owes is SumGroup.ATTN_TP - else _attn_tp_slice_update_read, - scatters_residual=residual != residual_to, - read=read, - ), + partial(step, scatters_residual=residual != residual_to, read=read), None, ) if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}: @@ -507,7 +543,7 @@ def _select_entry_step( return (partial(cp_moves.gather, gather=on_chunk), None) if gathered == {TokenAxis.ATTN_TP}: # A complete input on each rank's slice, gathered over attention TP - # once it is read. + # once it is read, unless one of the read's own kernels gathers it. if ( owes is not None or residual != residual_to @@ -520,8 +556,13 @@ def _select_entry_step( pre_move=None, enters_stack=enters_stack, read=read, + read_gathers=( + getattr(read, "gathering_reads", ()) + if is_plain_add or arrives_written + else () + ), ), - attn_tp_gather_input, + gather_input, ) if ( residual_to != produced.layout @@ -572,11 +613,13 @@ def _select_entry_step( ): raise NotImplementedError(f"{produced=} {need=}") fused = tuple(f for f in fusions if f.completes is owes) + read_fused = _read_fusions(read, owes, is_plain_add, scatters=False) return ( partial( _reduce_update_read, gathers_residual=gathers_residual, fusions=tuple(f.run for f in fused), + read_fusions=read_fused, group=owes, read=read, ), @@ -625,6 +668,7 @@ def _select_exit_move( residual: Layout, to: Layout, cp_moves: Optional[CpMoves] = None, + attn_tp_gather: Optional[Callable] = None, ) -> ExitMove: """How the FFN output reaches the rows the layer hands on: by undoing the attention-DP gather (the FFN exit and finish run that step), or else the @@ -638,7 +682,13 @@ def _select_exit_move( if to.sharded == residual.sharded - {TokenAxis.ATTN_TP}: # Each rank's slice back to the attention's rows: write the output # into the residual, then gather over attention TP. - return ExitMove(partial(update_attn_tp_gather_output, update=update)) + return ExitMove( + partial( + update_attn_tp_gather_output, + update=update, + gather=attn_tp_gather, + ) + ) raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded if returned == {TokenAxis.ATTN_TP} and to in (residual, produced.layout): diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index ae7791994f..34a7e1f854 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -47,6 +47,10 @@ from sglang.srt.layers.layer_boundary.layout import ( batches_are_unpadded, is_dense_ffn_fully_dp, ) +from sglang.srt.layers.layer_boundary.ops import ( + attn_tp_gather_input, + update_attn_tp_gather_output, +) from sglang.srt.layers.layer_boundary.prepare import ( _attn_input_default, _attn_input_scattered, @@ -186,6 +190,7 @@ class StagePlan: finishes_directly=False, qkv_latent_func=None, fusions=None, + attn_tp_gather=None, ): self.norm = norm self.edges = dict(variants) @@ -229,11 +234,16 @@ class StagePlan: cp_moves=edges.cp_moves, enters_stack=enters_stack, attn_input_adapter=edges.attn_input_adapter, + attn_tp_gather=attn_tp_gather, ) out = ( ExitMove() if finishes_directly - else bind_exit(edges.outgoing, cp_moves=edges.cp_moves) + else bind_exit( + edges.outgoing, + cp_moves=edges.cp_moves, + attn_tp_gather=attn_tp_gather, + ) ) self.paths[variant] = StagePath( entry=entry, @@ -243,6 +253,15 @@ class StagePlan: returns_over_dp=out.returns_over_dp, writes_at_handoff=writes_at_handoff, ) + if attn_tp_gather is not None and not any( + getattr(path.output_move, "func", None) is update_attn_tp_gather_output + or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input + for path in self.paths.values() + ): + raise ValueError( + "an attention-TP gather is declared, but no batch variant " + "gathers over attention TP at this stage's boundaries" + ) @property def incoming_residual_rows(self): @@ -334,6 +353,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options): is_branch=declaration.prepared_from is not None, terminal=declaration.terminal, writes_at_handoff=declaration.writes_at_handoff, + attn_tp_gather=declaration.attn_tp_gather, finishes_directly=declaration.kind is StageKind.ATTENTION and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL, **options, diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index c194562daa..2e65f79aae 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -14,7 +14,7 @@ """Immutable stage declarations and bound input/output paths.""" from enum import Enum, auto -from typing import Callable, FrozenSet, Optional, Tuple +from typing import Callable, FrozenSet, Optional, Tuple, Union import msgspec import torch @@ -45,6 +45,10 @@ class ExitRows(Enum): ATTENTION = auto() TBO_SPLIT = auto() + # The rows the FFN ran on, also at the stack's end: an FFN on this rank's + # attention-TP slice leaves its output there, and the model's final read + # must read that slice and gather it. + SLICE = auto() class BatchVariant(Enum): @@ -139,6 +143,8 @@ class EdgeContract(msgspec.Struct, frozen=True): arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for arriving contributions. Empty means use produced.update's capability. The actual update object travels with the residual stream. + arrives_written: Whether the producer applies its update at its exit, + so the stream arrives written with no residual add pending. """ produced: OutputContract @@ -150,6 +156,7 @@ class EdgeContract(msgspec.Struct, frozen=True): residual_joins_sum: bool = False # Capabilities allowed to arrive from another layer, not its update object. arriving_plain_add: Tuple[bool, ...] = () + arrives_written: bool = False class FfnInputFusion(msgspec.Struct, frozen=True): @@ -168,6 +175,28 @@ class FfnInputFusion(msgspec.Struct, frozen=True): preserves_residual: Optional[Callable] = None +class ReadoutFusion(msgspec.Struct, frozen=True): + """A kernel a stage's read supplies (its ``completing_fusions``) that + completes the sum its input owes together with the residual add, for a + read that is not the residual's plain norm and so takes no FfnInputFusion. + + ``run(hidden_states, residual, forward_batch)`` returns the written stream, + the completed sum plus the residual, which the read then reads; or None + when it does not take the batch, before touching its inputs or starting a + collective.""" + + # The group whose sum it completes. + completes: SumGroup + run: Callable[..., Optional[Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]] + # Whether it completes the sum onto this rank's attention-TP slice of the + # rows (a reduce-scatter, given the whole residual or its slice) rather + # than on every row. + scatters: bool = False + # Whether it also does the read: run(hidden_states, residual, + # forward_batch, norm) then returns the read's (input, residual). + reads: bool = False + + class CpMoves(msgspec.Struct, frozen=True): """How a CP extend's rows reach an FFN that needs all of them and come back, chosen once for the kind of prefill CP: ``gather`` gathers the FFN input diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 4c1ebb8784..988fa4f4f9 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -7,7 +7,7 @@ import os import sys from dataclasses import dataclass, replace from types import MappingProxyType -from typing import Mapping, Optional +from typing import Callable, Mapping, Optional import msgspec @@ -260,6 +260,11 @@ class StageDeclaration: output_complete: Whether the FFN's compute completes its own output sum, so the exit owes none. For an FFN whose output sum is fused with a reduction its computation needs anyway. + attn_tp_gather: Optional implementation of the gather over attention + TP this stage's boundaries run: given this rank's contiguous slice + of the rows, it returns them all, in rank order, or None to leave + the gather to the boundary. Called on every batch, so it must be + CUDA-graph safe. previous: Declaration whose output this stage consumes, as the stack records it; across pipeline ranks it is built locally. prepared_from: Declaration whose already-read input a branch reuses. @@ -281,6 +286,7 @@ class StageDeclaration: exit_rows: Optional[ExitRows] = None writes_at_handoff: bool = False output_complete: bool = False + attn_tp_gather: Optional[Callable] = None # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -325,6 +331,7 @@ def declare_attn( gathers_attn_tp_input=True, output_transform=None, tp_group=SumGroup.ATTN_TP, + attn_tp_gather=None, ): """Declare attention or a mixer; construct its executable boundary later. @@ -339,6 +346,8 @@ def declare_attn( stage's input runs it, so no fused add + norm takes that input. Requires ALWAYS_PARTIAL. tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers. + attn_tp_gather: Implementation of this stage's gathers over attention + TP, tried before the boundary's own (see StageDeclaration). Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -356,6 +365,7 @@ def declare_attn( reduction=reduction, gathers_attn_tp_input=gathers_attn_tp_input, tp_group=tp_group, + attn_tp_gather=attn_tp_gather, ) @@ -369,6 +379,7 @@ def declare_ffn( dense_tp_size=None, exit_rows=None, output_complete=False, + attn_tp_gather=None, ): """Declare a dense or MoE FFN independently of its compute module. @@ -385,6 +396,8 @@ def declare_ffn( the adjacent FFN kinds and TBO configuration. output_complete: Whether the compute completes its own output sum, fused with a reduction it needs anyway; the exit then owes none. + attn_tp_gather: Implementation of this stage's gathers over attention + TP, tried before the boundary's own (see StageDeclaration). Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -398,6 +411,7 @@ def declare_ffn( dense_tp_size=dense_tp_size, exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), output_complete=output_complete, + attn_tp_gather=attn_tp_gather, ) @@ -423,8 +437,13 @@ def _resolve_stage(stage, variant, following=None): dense_tp_size=stage.dense_tp_size, output_complete=stage.output_complete, ) - if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION: + exit_rows = resolve_exit_rows(stage.exit_rows) + if exit_rows is ExitRows.ATTENTION: returned = attention + elif exit_rows is ExitRows.SLICE: + # The residual's rows during the FFN: this rank's slice for an FFN + # on its own rows, else the attention's. + returned = residual return declaration, residual, returned sp = variant is BatchVariant.SEQUENCE_PARALLEL scattered = variant is BatchVariant.INPUT_SCATTERED @@ -489,6 +508,7 @@ def _connect(producer, consumer, *, residual_from=None): exits, entries = {}, {} for variant in _active_variants(): _, attention, local, _ = _row_layouts(variant) + written = False if before is None: rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention owes = variant is BatchVariant.INPUT_SCATTERED @@ -548,6 +568,7 @@ def _connect(producer, consumer, *, residual_from=None): update=None, ) residual, capabilities = returned, (before.update.is_plain_add,) + written = before.update.applied_at_exit if after is None: continue decl, during, _ = _resolve_stage(after, variant) @@ -579,6 +600,7 @@ def _connect(producer, consumer, *, residual_from=None): during, residual_joins_sum=joins, arriving_plain_add=capabilities, + arrives_written=written, ) entries[variant] = edge if ( diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index 61d84e8156..5262ca17ca 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -91,10 +91,27 @@ def tp_slice(hidden_states, residual): return hidden_states, residual +class GatheredInput: + """A stage input its read already gathered over attention TP, which the + entry's gather then passes on as it is.""" + + __slots__ = ("value",) + + def __init__(self, value: torch.Tensor): + self.value = value + + def attn_tp_gather_input( hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], forward_batch: ForwardBatch, + gather: Optional[Callable] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """Gather a stage input from this rank's attention-TP slice of the rows, + trying the stage's ``gather`` first for a single tensor (see + attn_tp_gather_with); an input its read already gathered (GatheredInput) + passes through.""" + if isinstance(hidden_states, GatheredInput): + return hidden_states.value parallel = get_parallel() if isinstance(hidden_states, tuple): gathered_hidden_states = [] @@ -118,7 +135,7 @@ def attn_tp_gather_input( gathered_hidden_states.append(output) return tuple(gathered_hidden_states) - return attn_tp_gather(hidden_states) + return attn_tp_gather_with(hidden_states, gather) def attn_tp_reduce_scatter( @@ -328,10 +345,22 @@ def update_attn_tp_gather_output( residual: torch.Tensor, forward_batch: ForwardBatch, update: ResidualUpdate = PLAIN_ADD, + gather: Optional[Callable] = None, **kwargs, ): + """Write the output into the residual on this rank's slice, then gather it + over attention TP (see attn_tp_gather_with).""" hidden_states = update.update(hidden_states, residual) - return attn_tp_gather(hidden_states), None + return attn_tp_gather_with(hidden_states, gather), None + + +def attn_tp_gather_with( + hidden_states: torch.Tensor, gather: Optional[Callable] +) -> torch.Tensor: + """Gather this rank's attention-TP slice of the rows into all of them: in + the stage's own ``gather`` when it takes the batch, else the boundary's.""" + gathered = gather(hidden_states) if gather is not None else None + return attn_tp_gather(hidden_states) if gathered is None else gathered def residual_slice_output( diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 983949a187..66c6c8b281 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -40,6 +40,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( get_attn_tp_context, tp_gather, ) +from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion from sglang.srt.layers.layer_boundary.layout import ( SumGroup, _cp_shard_token_rows, @@ -66,6 +67,7 @@ if _is_npu: from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache from sglang.srt.layers.layer_boundary.ops import ( + GatheredInput, attn_tp_all_reduce, attn_tp_reduce_scatter, dp_gather, @@ -74,6 +76,25 @@ from sglang.srt.layers.layer_boundary.ops import ( ) +def _run_read_fusions( + read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call +): + """The first of the read's own kernels (ReadoutFusion) that takes the + batch: one that completes the sum with the residual add hands the written + stream to ``read``; one that also reads returns the read. None when every + kernel declines.""" + for fused in read_fusions: + if fused.reads: + result = fused.run(hidden_states, residual, forward_batch, norm) + if result is not None: + return result + continue + stream = fused.run(hidden_states, residual, forward_batch) + if stream is not None: + return read.read(stream, norm, **call) + return None + + def _reduce_update_read( hidden_states: torch.Tensor, residual: torch.Tensor, @@ -83,6 +104,7 @@ def _reduce_update_read( cache=None, gathers_residual: bool, fusions: Tuple[Callable, ...], + read_fusions: Tuple[ReadoutFusion, ...] = (), group: SumGroup = SumGroup.ATTN_TP, read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, @@ -90,11 +112,24 @@ def _reduce_update_read( post_residual_addition: Optional[torch.Tensor] = None, ): """Complete the sum the input owes over ``group`` on the rows it is on, - unless one of ``fusions`` does it with the residual add and the norm, then - write it into the residual and read the input (an attention's read takes + unless one of the read's ``read_fusions`` does it with the residual add, + or one of ``fusions`` with the residual add and the norm, then write it + into the residual and read the input (an attention's read takes ``quant_format`` and ``post_residual_addition``).""" if gathers_residual: residual = update.gather_residual_attn_tp(residual) + result = _run_read_fusions( + read_fusions, + hidden_states, + residual, + forward_batch, + norm, + read=read, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) + if result is not None: + return result for fused in fusions: result = fused(hidden_states, residual, forward_batch) if result is not None: @@ -315,6 +350,7 @@ def _update_read( pre_move: Optional[Callable], enters_stack: bool, read: ResidualReadout, + read_gathers: Tuple[Callable, ...] = (), update: ResidualUpdate = PLAIN_ADD, quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, @@ -322,12 +358,19 @@ def _update_read( """Complete what the input owes by construction (``pre_move``), then write the previous stage's output into the residual and read the stage's input with ``norm``. The layer stack's first stage (``enters_stack``) - starts its residual from its input.""" + starts its residual from its input. The first of ``read_gathers`` that + takes the batch reads and gathers the input itself; its result is wrapped + in GatheredInput so the entry's gather passes it through.""" enters = residual is None and enters_stack if pre_move is not None: hidden_states, residual = pre_move(hidden_states, residual) if enters: hidden_states, residual = read.init_residual(hidden_states), None + for fused in read_gathers: + result = fused(hidden_states, residual, norm) + if result is not None: + gathered, residual = result + return GatheredInput(gathered), residual if residual is None: # The previous layer already wrote its output into the residual. return read.read(hidden_states, norm, quant_format) @@ -349,9 +392,19 @@ def _attn_tp_reduce_scatter_update_read( *, cache=None, scatters_residual: bool, + read_fusions: Tuple[ReadoutFusion, ...] = (), read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, ): + """Complete the attention-TP sum onto this rank's slice, slice the residual + the same way, then write the output into it and read the input; the first + of the read's ``read_fusions`` that takes the batch does all of this + instead.""" + result = _run_read_fusions( + read_fusions, hidden_states, residual, forward_batch, norm, read=read + ) + if result is not None: + return result hidden_states = attn_tp_reduce_scatter(hidden_states) if scatters_residual and residual is not None: residual = update.slice_residual_attn_tp(residual) diff --git a/python/sglang/srt/layers/layer_boundary/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py index c24705c6f6..de3ac0db81 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/residual/__init__.py @@ -70,6 +70,13 @@ class ResidualReadout(Protocol): is_plain_norm: Read is normalization/optional quantization without changing the residual, allowing compatible fused add+norm implementations. reads_before_dp_gather: Preserve this read on source rows before a DP gather. + completing_fusions: Optional. ReadoutFusion kernels that complete the + sum the input owes with the residual add ahead of this read, tried + before the boundary's own all-reduce or reduce-scatter. + gathering_reads: Optional. Callables ``(hidden_states, residual, + norm)`` that read the input on this rank's attention-TP slice and + gather it over attention TP, returning ``(input, residual)``, or + None when they do not take the batch. init_residual initializes the stack residual. read consumes an already-written residual; update_and_read first applies the actual diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py index f88f6ced43..e3f6fa49c7 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -14,11 +14,14 @@ """A residual read that aggregates a bank of snapshots of itself.""" from dataclasses import dataclass -from typing import Optional +from typing import Optional, Tuple import torch from sglang.srt.layers.attn_residual import AttnResidual +from sglang.srt.layers.layer_boundary import ops +from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion +from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.residual import LayerResidualOps from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD from sglang.srt.runtime_context import get_parallel @@ -87,6 +90,12 @@ class AttnBankState: ffn_score_proj: torch.nn.Module ffn_score_norm: torch.nn.Module writes_block: bool = False + # Kernels that complete the attention output's sum with the pending add + # ahead of the FFN read. + ffn_input_fusions: Tuple[ReadoutFusion, ...] = () + # A read on this rank's attention-TP slice first tries the bank's kernels + # that also run the collective around it. + fuses_slice_collectives: bool = False def _aggregate( self, contribution, residual, norm, *, score_proj, score_norm, write @@ -131,6 +140,42 @@ class AttnBankState: write=False, ) + def reduce_scatter_and_read_ffn_input( + self, hidden_states, residual, forward_batch, norm + ): + parallel = get_parallel() + rows, rest = divmod(hidden_states.shape[0], parallel.attn_tp_size) + if rows == 0 or rest: + return None + start = parallel.attn_tp_rank * rows + return self.bank.require().forward_sp_reduce_scatter( + hidden_states, + residual, + self.ffn_score_proj, + self.ffn_score_norm, + norm, + rows=slice(start, start + rows), + ) + + def read_and_gather_attn_input(self, contribution, residual, norm): + bank = self.bank.require() + rows = _bank_rows(bank, contribution) + if rows is None: + return None + read = bank.forward_sp_all_gather( + contribution, + residual, + self.attn_score_proj, + self.attn_score_norm, + norm, + rows=rows, + write=self.writes_block, + ) + if read is None: + return None + normed, residual = read + return normed, None if self.writes_block else residual + def residual_ops(self) -> LayerResidualOps: return LayerResidualOps( attn_readout=_AttnReadout(self), @@ -171,6 +216,13 @@ class _AttnReadout(_BankReadout): """The attention input: the bank aggregation and this layer's input norm. A write layer snapshots the residual this read forms.""" + @property + def gathering_reads(self): + state = self.state + return ( + (state.read_and_gather_attn_input,) if state.fuses_slice_collectives else () + ) + def read(self, residual, norm, quant_format="", post_residual_addition=None): self._reject(None, quant_format) if post_residual_addition is not None: @@ -188,6 +240,19 @@ class _FfnReadout(_BankReadout): """The FFN input: the attention output's add folded into the bank aggregation, and this layer's post-attention norm.""" + @property + def completing_fusions(self) -> Tuple[ReadoutFusion, ...]: + state = self.state + if not state.fuses_slice_collectives: + return state.ffn_input_fusions + reads = ReadoutFusion( + SumGroup.ATTN_TP, + state.reduce_scatter_and_read_ffn_input, + scatters=True, + reads=True, + ) + return (reads, *state.ffn_input_fusions) + def read(self, residual, norm, quant_format="", post_residual_addition=None): # The attention read of a write layer leaves the residual empty, so the # attention output alone is the head. @@ -202,22 +267,40 @@ class _FfnReadout(_BankReadout): class AttnBankOutputRead: """The layer stack's terminal read: the bank aggregation with the output side's scoring parameters, then the final norm. A callable final norm for - `residual_batch.final_norm`.""" + `residual_batch.final_norm`. - def __init__(self, bank: AttnBank, score_proj, score_norm, norm): + A stack that ends on this rank's attention-TP slice of the rows is read + there, against the bank rows this rank wrote, and what it read is + gathered: by the bank's kernel that does both when it takes the batch, + else by ``attn_tp_gather`` when that does, else over the attention-TP + group.""" + + def __init__( + self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None + ): self.bank = bank self.score_proj = score_proj self.score_norm = score_norm self.norm = norm + self.attn_tp_gather = attn_tp_gather def __call__(self, hidden_states, residual=None): bank = self.bank.require() - normed, head = bank.forward( - hidden_states, - residual, - self.score_proj, - self.score_norm, - self.norm, - rows=_bank_rows(bank, hidden_states), + rows = _bank_rows(bank, hidden_states) + read_args = (hidden_states, residual, self.score_proj, self.score_norm) + read = ( + None + if rows is None + else bank.forward_sp_all_gather(*read_args, self.norm, rows=rows) ) + if read is None: + read = bank.forward(*read_args, self.norm, rows=rows) + if rows is not None: + read = (self._gather(read[0]), read[1]) + normed, head = read return normed if residual is None else (normed, head) + + def _gather(self, normed): + gathered = None if self.attn_tp_gather is None else self.attn_tp_gather(normed) + # The output outlives the layer stack. + return ops.attn_tp_gather(normed, owned=True) if gathered is None else gathered diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index ba0aba9a24..3a4378d845 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -51,6 +51,8 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.layers.layer_boundary import ( ExitRows, + ReadoutFusion, + SumGroup, append_stages, declare_attn, declare_ffn, @@ -142,7 +144,9 @@ from sglang.srt.runtime_context import ( get_exec, get_parallel, get_platform, + get_spec, ) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import is_hip, is_npu, make_pp_layers from sglang.srt.utils.common import ( BumpAllocator, @@ -428,6 +432,48 @@ def _o_proj_takes_output(o_proj: RowParallelLinear) -> bool: return getattr(o_proj.quant_method, "apply_into", None) is not None +def _k3_all_reduce_add(hidden_states, residual, forward_batch): + """o_proj's attention-TP sum with the pending residual add in K3's fused + all-reduce, for an output o_proj wrote into its multicast buffer.""" + if k3_ar_fusion.find_mc_ptr(hidden_states) is None: + return None + return k3_ar_fusion.all_reduce(hidden_states, residual) + + +def _k3_reduce_scatter_add(hidden_states, residual, forward_batch): + """o_proj's attention-TP sum onto this rank's slice with the pending + residual add, in K3's tuned reduce-scatter when it takes the batch.""" + return k3_sp_collective.reduce_scatter_res(hidden_states, residual) + + +def _route_sp_o_proj_output(o_proj: RowParallelLinear) -> None: + """o_proj under SP-MoE.""" + # o_proj emits TP-partial sums, which the stage boundary completes. + o_proj.reduce_results = False + if k3_sp_collective.enabled(): + # The table selects NVLS pull RS for larger token buckets. + # Only those o_proj outputs come from the persistent symmetric + # buffer; small push RS keeps the regular graph allocator. + _sp_inner_o_proj_forward = o_proj.forward + + def _sp_o_proj_forward(x, *args, **kwargs): + output_rows = x.shape[0] + if k3_sp_collective.requires_symmetric_rs( + output_rows, x.device, x.element_size() + ): + output = k3_sp_collective.get_o_proj_output_buffer( + output_rows, x.dtype, o_proj.output_size + ) + result = _sp_inner_o_proj_forward( + x, *args, output_tensor=output[: x.shape[0]], **kwargs + ) + k3_sp_collective.register_o_proj_output(result[0], output) + return result + return _sp_inner_o_proj_forward(x, *args, **kwargs) + + o_proj.forward = _sp_o_proj_forward + + def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Tensor: """Symmetric storage for o_proj's TP-partial output; the fused attention all-reduce reduces it in place.""" @@ -1546,6 +1592,7 @@ class KimiK3DeltaAttention(nn.Module): all_reduce_fusion: bool = False, bfa_alt_stream: Optional[torch.cuda.Stream] = None, reduce_results: bool = True, + sp_moe: bool = False, **kwargs, ) -> None: super().__init__() @@ -1818,7 +1865,7 @@ class KimiK3DeltaAttention(nn.Module): # the fused AR reduces o_proj's output in place out of a symmetric # buffer, which needs the GEMM to write into caller-owned storage self.all_reduce_fusion = False - self.o_proj.reduce_results = True + self.o_proj.reduce_results = reduce_results self.o_proj.use_dp_attention_reduce = True conv_weights = self.qkv_conv1d.weight.squeeze(1) bias = self.qkv_conv1d.bias @@ -1841,6 +1888,8 @@ class KimiK3DeltaAttention(nn.Module): # Set by _prepare_fused_decode() once weights are loaded. self._kda_fused_decode_ready = False self._kda_hip_fused_decode_ready = False + if sp_moe: + _route_sp_o_proj_output(self.o_proj) def forward_qkvbfg(self, hidden_states: torch.Tensor): qkv, _ = self.qkv_proj(hidden_states) @@ -2200,6 +2249,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): alt_stream: Optional[torch.cuda.Stream] = None, gate_alt_stream: Optional[torch.cuda.Stream] = None, reduce_results: bool = True, + sp_moe: bool = False, ) -> None: # ModelSlim can quantize K3 latent projections while still storing # MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V. @@ -2276,7 +2326,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): # the fused AR reduces o_proj's output in place out of a symmetric # buffer, which needs the GEMM to write into caller-owned storage self.all_reduce_fusion = False - self.o_proj.reduce_results = True + self.o_proj.reduce_results = reduce_results self.o_proj.use_dp_attention_reduce = True if self.all_reduce_fusion: # Hand the GEMM a slice of the persistent symmetric buffer @@ -2350,6 +2400,8 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): return _orig_o_proj_forward(x, *args, **kwargs) self.o_proj.forward = _gated_o_proj_forward + if sp_moe: + _route_sp_o_proj_output(self.o_proj) @staticmethod def _split_kv_b_weight_loader(param, loaded_weight) -> None: @@ -2445,17 +2497,34 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool: ) -def _uses_stage_boundaries(config: KimiLinearConfig) -> bool: - """Whether the layers build stage boundaries, which is the same for every - layer of a stack. The layer's own communication still runs for: an - attention-residual bank whose o_proj all-reduce is fused with the pending - add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned - SP collectives, which the sharded carry also needs.""" - if _fuses_attn_all_reduce(config): +def _carries_bank_slices(config: KimiLinearConfig) -> bool: + """Whether consecutive SP-MoE layers keep the stream and the + attention-residual bank on each rank's attention-TP shard of the rows + (SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then + gathers what it read. A rank then holds only its own rows of the bank, so + neither a later pipeline rank nor a draft model's capture of the target's + hidden states can take them; it is decided at construction. Nor can a + dense layer after an SP-MoE one, whose FFN reads every row of the bank: + every layer from the first MoE layer on must be one.""" + if not ( + _shards_moe_rows() + and config.attn_res_block_size is not None + and k3_sp_collective.enabled() + and envs.SGLANG_K3_SP_ATTN_RES.get() + and get_parallel().pp_size == 1 + ): return False - if not _shards_moe_rows(): - return True - return config.attn_res_block_size is None or not k3_sp_collective.enabled() + moe = [_is_moe_layer(config, idx) for idx in range(config.num_hidden_layers)] + if True not in moe or not all(moe[moe.index(True) :]): + return False + spec = SpeculativeAlgorithm.from_string(get_spec().speculative_algorithm) + return not (spec.is_eagle3() or spec.is_dflash_family()) + + +def _uses_stage_boundaries(config: KimiLinearConfig) -> bool: + """Whether the layers build stage boundaries, which every configuration + does.""" + return True class KimiK3DecoderLayer(nn.Module): @@ -2500,6 +2569,7 @@ class KimiK3DecoderLayer(nn.Module): prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, reduce_results=not self._stage_boundaries, + sp_moe=self._sp_moe, # Shared with the MLA gate stream: KDA and MLA layers never # run concurrently within one forward, so the stream is free. bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None), @@ -2512,6 +2582,7 @@ class KimiK3DecoderLayer(nn.Module): prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, reduce_results=not self._stage_boundaries, + sp_moe=self._sp_moe, alt_stream=alt_streams[1] if alt_streams is not None else None, gate_alt_stream=alt_streams[2] if alt_streams is not None else None, ) @@ -2571,35 +2642,6 @@ class KimiK3DecoderLayer(nn.Module): prefix=f"{prefix}.mlp_res_proj", ) - if self._sp_moe: - # o_proj emits TP-partial sums; _finish_attn_reduce completes the - # reduction (RS on the clean attn-res path, AR on fallbacks). - o_proj = getattr(self.self_attn, "o_proj", None) - assert o_proj is not None, "SP-MoE requires attention exposing o_proj" - o_proj.reduce_results = False - if k3_sp_collective.enabled(): - # The table selects NVLS pull RS for larger token buckets. - # Only those o_proj outputs come from the persistent symmetric - # buffer; small push RS keeps the regular graph allocator. - _sp_inner_o_proj_forward = o_proj.forward - - def _sp_o_proj_forward(x, *args, **kwargs): - output_rows = x.shape[0] - if k3_sp_collective.requires_symmetric_rs( - output_rows, x.device, x.element_size() - ): - output = k3_sp_collective.get_o_proj_output_buffer( - output_rows, x.dtype, o_proj.output_size - ) - result = _sp_inner_o_proj_forward( - x, *args, output_tensor=output[: x.shape[0]], **kwargs - ) - k3_sp_collective.register_o_proj_output(result[0], output) - return result - return _sp_inner_o_proj_forward(x, *args, **kwargs) - - o_proj.forward = _sp_o_proj_forward - # A latent MoE on this rank's rows adds the residual in its tail add and # writes the next stream itself; one gathered over attention DP returns # its output to this rank's tokens first. @@ -2611,56 +2653,98 @@ class KimiK3DecoderLayer(nn.Module): and not (self._dp_attention and not self.mlp._ep_a2a) ) if self._stage_boundaries: - # Under attention DP the FFN input is read on this rank's rows - # once the attention's sum is complete, then gathered. - attn_ops = {} - ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) - if self.use_attn_residuals: - bank_ops = AttnBankState( - attn_bank, - self.self_attention_res_proj, - self.self_attention_res_norm, - self.mlp_res_proj, - self.mlp_res_norm, - writes_block=self.is_block_write_layer, - ).residual_ops() - attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update) - ffn_ops = dict( - read=bank_ops.ffn_readout, - update=( - REPLACE_AT_EXIT - if self._ffn_writes_stream - else bank_ops.ffn_update - ), - ) - self.attn_boundary, self.ffn_boundary = append_stages( - (declare_attn(**attn_ops), self.input_layernorm), - ( - declare_ffn( - **ffn_ops, - sparse=self._is_moe_layer, - next_layer_sparse=_is_moe_layer(config, layer_idx + 1), - # The TP width the dense MLP is built with. - dense_tp_size=( - None - if self._is_moe_layer - else get_group_rank_size(self.mlp.down_proj.tp_group)[1] - ), - # SP-MoE runs on this rank's attention-TP shard of the - # rows; on the bank path, whose reads write the bank on - # every row, its output returns to all of them. - exit_rows=( - ExitRows.ATTENTION - if self._sp_moe and self.use_attn_residuals - else None - ), - # A latent MoE completes its output sum together with - # the latent reduction its norm needs. - output_complete=self._is_moe_layer and self.mlp.use_latent_moe, - ), - self.post_attention_layernorm, + self._declare_stages(config, layer_idx, attn_bank) + + def _declare_stages(self, config, layer_idx, attn_bank): + """Declare this layer's attention and FFN stages: the reads and updates + around them, the rows the FFN takes and leaves, and the kernels K3 + supplies for the bank path.""" + # Under attention DP the FFN input is read on this rank's rows + # once the attention's sum is complete, then gathered. + attn_ops = {} + ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) + carries = _carries_bank_slices(config) + bank_sp_moe = self._sp_moe and self.use_attn_residuals + tuned_gather = ( + k3_sp_collective.all_gather if k3_sp_collective.enabled() else None + ) + if self.use_attn_residuals: + bank_ops = AttnBankState( + attn_bank, + self.self_attention_res_proj, + self.self_attention_res_norm, + self.mlp_res_proj, + self.mlp_res_norm, + writes_block=self.is_block_write_layer, + ffn_input_fusions=self._ffn_input_fusions(), + fuses_slice_collectives=self._sp_moe + and k3_sp_collective.enabled() + and envs.SGLANG_K3_SP_ATTN_RES.get(), + ).residual_ops() + attn_ops = dict( + read=bank_ops.attn_readout, + update=bank_ops.attn_update, + # An input read on the shard an SP-MoE layer left is gathered + # in K3's tuned all-gather when it takes the batch. + attn_tp_gather=( + tuned_gather + if carries and _is_moe_layer(config, layer_idx - 1) + else None ), ) + ffn_ops = dict( + read=bank_ops.ffn_readout, + update=( + REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update + ), + ) + self.attn_boundary, self.ffn_boundary = append_stages( + (declare_attn(**attn_ops), self.input_layernorm), + ( + declare_ffn( + **ffn_ops, + sparse=self._is_moe_layer, + next_layer_sparse=_is_moe_layer(config, layer_idx + 1), + # The TP width the dense MLP is built with. + dense_tp_size=( + None + if self._is_moe_layer + else get_group_rank_size(self.mlp.down_proj.tp_group)[1] + ), + # SP-MoE runs on this rank's attention-TP shard of the + # rows; on the bank path, whose reads write the bank on + # every row, its output returns to all of them, unless + # the bank stays on the shard, read there to the end. + exit_rows=( + (ExitRows.SLICE if carries else ExitRows.ATTENTION) + if bank_sp_moe + else None + ), + # A latent MoE completes its output sum together with + # the latent reduction its norm needs. + output_complete=self._is_moe_layer and self.mlp.use_latent_moe, + # The bank path's gather back to every row, in K3's + # tuned all-gather when it takes the batch. + attn_tp_gather=( + tuned_gather if bank_sp_moe and not carries else None + ), + ), + self.post_attention_layernorm, + ), + ) + + def _ffn_input_fusions(self): + """K3's kernels that complete the attention output's sum with the + pending residual add ahead of the bank's FFN read: the fused + all-reduce on every row, or the tuned reduce-scatter onto this rank's + attention-TP shard (SP-MoE).""" + if self.all_reduce_fusion: + return (ReadoutFusion(SumGroup.ATTN_TP, _k3_all_reduce_add),) + if self._sp_moe and k3_sp_collective.enabled(): + return ( + ReadoutFusion(SumGroup.ATTN_TP, _k3_reduce_scatter_add, scatters=True), + ) + return () def _finish_attn_reduce( self, @@ -2949,6 +3033,7 @@ class KimiK3LinearModel(nn.Module): self.dspark_layers_to_capture: Optional[list[int]] = None self._dp_attention = is_dp_attention_enabled() self._stage_boundaries = _uses_stage_boundaries(config) + self.carries_bank_slices = _carries_bank_slices(config) if self.pp_group.is_first_rank: embedding_quant_config = ( @@ -3016,6 +3101,11 @@ class KimiK3LinearModel(nn.Module): self.output_attn_res_proj, self.output_attn_res_norm, self.norm, + attn_tp_gather=( + k3_sp_collective.all_gather + if self.carries_bank_slices + else None + ), ) ) else: @@ -3359,6 +3449,12 @@ class KimiK3LinearForCausalLM(nn.Module): raise ValueError( "DSPARK requires explicit layer_ids for aux hidden capture." ) + if self.model.carries_bank_slices: + raise RuntimeError( + "the model keeps each rank's shard of the rows across SP-MoE " + "layers (SGLANG_K3_SP_ATTN_RES), which leaves no layer output " + "to capture whole" + ) self.capture_aux_hidden_states = True self.model.dspark_layers_to_capture = list(layer_ids) diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index e7c9083f44..a95bb4daf3 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -14,6 +14,7 @@ from sglang.srt.layers.layer_boundary import ( BatchVariant, ExitRows, ProducerReduction, + ReadoutFusion, append_stages, declare_attn, declare_ffn, @@ -26,6 +27,11 @@ from sglang.srt.layers.layer_boundary.ops import ( keep_output, update_attn_tp_gather_output, ) +from sglang.srt.layers.layer_boundary.residual.add_norm import ( + PLAIN_ADD, + REPLACE_AT_EXIT, + NormQuantReadout, +) from sglang.srt.layers.rotary_embedding import factory as rope_factory from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.utils.common import is_building_neighbour_layer, make_layers @@ -451,6 +457,171 @@ class TestUnpaddedBatches(CustomTestCase): self.assertNotIn(BatchVariant.UNPADDED, ffn.plan.paths) +class TestSliceExitRows(CustomTestCase): + """An FFN on this rank's attention-TP slice that declares ExitRows.SLICE + leaves its output there, the stack's last one too, where the final read + then takes it; the next layer's attention reads it there and gathers what + it read, in the declared gather when that takes the batch.""" + + def build(self, exit_rows, *, disable_attn_tp_gather=False, gather=None): + parallel = fixture.parallel_of( + attn_dp=1, attn_tp=2, disable_attn_tp_gather=disable_attn_tp_gather + ) + with fixture.planning(parallel, a2a=True, boundary_reduction="ar"): + with layer_stack(): + return [ + stage + for idx in range(2) + for stage in append_stages( + ( + declare_attn(attn_tp_gather=gather if idx else None), + fixture.Norm(), + ), + ( + declare_ffn( + sparse=True, + next_layer_sparse=True, + exit_rows=exit_rows, + ), + fixture.Norm(), + ), + ) + ] + + def test_the_last_ffn_stays_on_its_slice(self): + for exit_rows in (ExitRows.SLICE, None): + with self.subTest(exit_rows=exit_rows): + last = self.build(exit_rows)[-1].plan.paths[BatchVariant.ORDINARY] + if exit_rows is ExitRows.SLICE: + self.assertIs(last.output_move, keep_output) + else: + self.assertIs(last.output_move.func, update_attn_tp_gather_output) + + def test_an_unpadded_batch_leaves_the_last_ffn_on_the_attention_rows(self): + last = self.build(ExitRows.SLICE, disable_attn_tp_gather=True)[-1] + unpadded = last.plan.paths[BatchVariant.UNPADDED] + self.assertEqual(unpadded.output.layout.sharded, frozenset()) + self.assertIs(unpadded.output_move, keep_output) + + def test_the_next_attention_gathers_in_the_declared_gather(self): + def gather(hidden_states): + return None + + _, _, attention, _ = self.build(ExitRows.SLICE, gather=gather) + move = attention.plan.paths[BatchVariant.ORDINARY].entry.input_move + self.assertIs(move.func, attn_tp_gather_input) + self.assertIs(move.keywords["gather"], gather) + + def test_a_read_that_gathers_needs_a_plain_add_or_a_written_stream(self): + def gather_read(hidden_states, residual, norm): + return None + + class GatheringRead(NormQuantReadout): + gathering_reads = (gather_read,) + + class Scaled: + """Neither a plain add nor applied at the producer's exit.""" + + is_plain_add = False + applied_at_exit = False + outlives_layer = True + + # One pipeline rank: an update other than a plain add can't cross one. + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1) + for update, binds in ( + (PLAIN_ADD, True), + (REPLACE_AT_EXIT, True), + (Scaled(), False), + ): + with self.subTest(update=type(update).__name__): + with ( + fixture.planning(parallel, a2a=True, boundary_reduction="ar"), + layer_stack(), + ): + stages = [ + stage + for _ in range(2) + for stage in append_stages( + (declare_attn(read=GatheringRead()), fixture.Norm()), + ( + declare_ffn( + sparse=True, + next_layer_sparse=True, + update=update, + exit_rows=ExitRows.SLICE, + ), + fixture.Norm(), + ), + ) + ] + # The stack binds its stages when it closes. + entry = stages[2].plan.paths[BatchVariant.ORDINARY].entry + self.assertEqual( + entry.prepare.keywords["step"].keywords["read_gathers"], + (gather_read,) if binds else (), + ) + + +class TestReadKernels(CustomTestCase): + """The entry steps around a read's own kernels: one that reads and gathers + hands its input to the entry's gather as already gathered, and one that + completes the sum and reads returns the read.""" + + def test_a_read_that_gathers_passes_through_the_entry_gather(self): + gathered = torch.ones(4, 8) + + class Read: + def read(self, residual, norm, quant_format=""): + return residual * 2, residual + + def takes(hidden_states, residual, norm): + return gathered, None + + def declines(hidden_states, residual, norm): + return None + + shard = torch.randn(2, 8) + common = dict(pre_move=None, enters_stack=False, read=Read()) + got, residual = boundary_prepare._update_read( + shard, None, None, None, read_gathers=(declines, takes), **common + ) + self.assertIs(attn_tp_gather_input(got, None), gathered) + self.assertIsNone(residual) + got, residual = boundary_prepare._update_read( + shard, None, None, None, read_gathers=(declines,), **common + ) + torch.testing.assert_close(got, shard * 2, rtol=0, atol=0) + + def test_a_kernel_that_reads_returns_the_read(self): + read_result = (torch.zeros(2, 8), torch.ones(2, 8)) + + def reads(takes): + def run(hidden_states, residual, forward_batch, norm): + return read_result if takes else None + + return ReadoutFusion(SumGroup.ATTN_TP, run, scatters=True, reads=True) + + class Read: + def update_and_read(self, update, hidden_states, residual, norm): + return "boundary", residual + + class Update: + def slice_residual_attn_tp(self, residual): + return residual[:2] + + call = dict(scatters_residual=True, read=Read(), update=Update()) + hidden, residual = torch.randn(4, 8), torch.randn(4, 8) + with patch.object(boundary_prepare, "attn_tp_reduce_scatter", lambda h: h[:2]): + got = boundary_prepare._attn_tp_reduce_scatter_update_read( + hidden, residual, None, None, read_fusions=(reads(True),), **call + ) + self.assertIs(got, read_result) + got, _ = boundary_prepare._attn_tp_reduce_scatter_update_read( + hidden, residual, None, None, read_fusions=(reads(False),), **call + ) + self.assertEqual(got, "boundary") + + class TestDenseFfnOverAttentionTp(CustomTestCase): """A dense FFN sharded over attention TP under attention DP computes on the attention's rows and sums over attention TP, with no DP move.""" diff --git a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py index 84398bfae6..7ed43787bc 100644 --- a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py @@ -12,11 +12,14 @@ from sglang.srt.layers import attn_residual from sglang.srt.layers.layer_boundary import ( BatchVariant, ExitRows, + ReadoutFusion, + SumGroup, append_stages, declare_attn, declare_ffn, layer_stack, ) +from sglang.srt.layers.layer_boundary import ops as comm_moves from sglang.srt.layers.layer_boundary import prepare as comm_ops from sglang.srt.layers.layer_boundary.ops import update_attn_tp_gather_output from sglang.srt.layers.layer_boundary.residual import attn_bank @@ -272,6 +275,73 @@ class TestAttnBankResidualOps(CustomTestCase): self.assertTrue(readout.reads_before_dp_gather) +class TestAttnBankFusedAllReduce(CustomTestCase): + """A bank's FFN read can supply a kernel that completes the attention + output's sum together with the pending residual add on every row; the + read then aggregates the stream that kernel wrote.""" + + def setUp(self): + patcher = patch.object(attn_residual, "_mix_fused", _mix) + patcher.start() + self.addCleanup(patcher.stop) + + @staticmethod + def fusion(takes): + def run(hidden, residual, forward_batch): + if not takes: + return None + return hidden if residual is None else hidden + residual + + return ReadoutFusion(SumGroup.ATTN_TP, run) + + @staticmethod + def ops(bank, fusions): + layer = LAYER_LIST[1] + return AttnBankState( + bank, + layer.attn_proj, + layer.attn_score_norm, + layer.ffn_proj, + layer.ffn_score_norm, + ffn_input_fusions=fusions, + ).residual_ops() + + def test_the_fused_stream_reads_as_the_entry_would(self): + hidden = torch.randn(TOKENS, HIDDEN, generator=torch.Generator().manual_seed(3)) + residual = torch.randn( + TOKENS, HIDDEN, generator=torch.Generator().manual_seed(4) + ) + results = [] + for takes in (True, False): + holder = AttnBank() + # A bank with one snapshot, as after the first block. + holder.open(torch.randn(TOKENS, HIDDEN), 2).write( + torch.randn(TOKENS, HIDDEN, generator=torch.Generator().manual_seed(5)) + ) + ops = self.ops(holder, (self.fusion(takes),)) + with patch.object( + comm_ops, "attn_tp_all_reduce", side_effect=lambda h, fb, **kw: h + ) as reduce: + results.append( + comm_ops._reduce_update_read( + hidden.clone(), + residual.clone(), + None, + LAYER_LIST[1].post_norm, + gathers_residual=False, + fusions=(), + read_fusions=ops.ffn_readout.completing_fusions, + read=ops.ffn_readout, + update=ops.ffn_update, + ) + ) + # Only a kernel that declines the batch leaves the sum to the + # boundary's own all-reduce. + self.assertEqual(reduce.called, not takes) + for fused, unfused in zip(*results): + torch.testing.assert_close(fused, unfused, rtol=0, atol=0) + + class TestAttnBankSpMoeStages(CustomTestCase): """A latent MoE dispatched over an a2a backend with attention TP runs on this rank's shard of the rows: the FFN's entry reduce-scatters the @@ -322,6 +392,198 @@ class TestAttnBankSpMoeStages(CustomTestCase): move = ffn.plan.paths[BatchVariant.ORDINARY].output_move self.assertIs(move.func, update_attn_tp_gather_output) + def test_the_exit_gather_falls_back_when_its_kernel_declines(self): + class Update: + def update(self, hidden_states, residual): + return hidden_states + 1 + + shard = torch.zeros(2, HIDDEN) + with patch.object(comm_moves, "attn_tp_gather", lambda h: torch.cat([h, h])): + tuned, _ = update_attn_tp_gather_output( + shard, + None, + None, + update=Update(), + gather=lambda h: torch.full((4, HIDDEN), 7.0), + ) + fallback, _ = update_attn_tp_gather_output( + shard, None, None, update=Update(), gather=lambda h: None + ) + torch.testing.assert_close(tuned, torch.full((4, HIDDEN), 7.0)) + torch.testing.assert_close(fallback, torch.ones(4, HIDDEN)) + + def test_the_shard_entry_reads_the_kernels_stream_as_its_own(self): + hidden = torch.randn(4, HIDDEN, generator=torch.Generator().manual_seed(8)) + residual = torch.randn(4, HIDDEN, generator=torch.Generator().manual_seed(9)) + + class Update: + is_plain_add = True + + def slice_residual_attn_tp(self, residual): + return residual[:2] + + def scatter_add(takes): + def run(hidden_states, residual, forward_batch): + return hidden_states[:2] + residual[:2] if takes else None + + return run + + results = [] + for takes in (True, False): + holder = AttnBank() + holder.open(torch.randn(4, HIDDEN), 2).write( + torch.randn(4, HIDDEN, generator=torch.Generator().manual_seed(10)) + ) + ops = AttnBankState( + holder, + LAYER_LIST[1].attn_proj, + LAYER_LIST[1].attn_score_norm, + LAYER_LIST[1].ffn_proj, + LAYER_LIST[1].ffn_score_norm, + ).residual_ops() + with ( + patch.object(attn_residual, "_mix_fused", _mix), + patch.object(comm_ops, "attn_tp_reduce_scatter", lambda h: h[:2]), + patch.object( + attn_bank, + "get_parallel", + lambda: SimpleNamespace(attn_tp_size=2, attn_tp_rank=0), + ), + ): + results.append( + comm_ops._attn_tp_reduce_scatter_update_read( + hidden.clone(), + residual.clone(), + None, + LAYER_LIST[1].post_norm, + scatters_residual=True, + read_fusions=( + ReadoutFusion( + SumGroup.ATTN_TP, scatter_add(takes), scatters=True + ), + ), + read=ops.ffn_readout, + update=Update(), + ) + ) + for fused, unfused in zip(*results): + torch.testing.assert_close(fused, unfused, rtol=0, atol=0) + + +class TestAttnBankOnShards(CustomTestCase): + """With the stream kept on this rank's attention-TP shard across MoE + layers, a write layer snapshots only this rank's rows of the bank. Reads + take the matching rows to the end, where the final read gathers what it + read; a read of every row after such a write is refused, since the other + ranks' rows of this bank are stale.""" + + def setUp(self): + for patcher in ( + patch.object(attn_residual, "_mix_fused", _mix), + # The second of two attention-TP ranks: rows 2 and 3 of 4. + patch.object( + attn_bank, + "get_parallel", + lambda: SimpleNamespace(attn_tp_size=2, attn_tp_rank=1), + ), + ): + patcher.start() + self.addCleanup(patcher.stop) + self.hidden = torch.randn( + 4, HIDDEN, generator=torch.Generator().manual_seed(11) + ) + + def open(self): + holder = AttnBank() + # One snapshot of every row, as after the first block. + holder.open(self.hidden, 3).write(self.hidden * 0.5) + return holder + + @staticmethod + def final_read(holder, gather=None): + return AttnBankOutputRead( + holder, OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM, attn_tp_gather=gather + ) + + def test_the_final_read_of_a_shard_gathers_what_it_read(self): + holder = self.open() + every_row = self.final_read(holder)(self.hidden) + + def all_gather(shard, *, owned=False): + # The output outlives the stack: it may not share a buffer. + self.assertTrue(owned) + return torch.cat([torch.zeros_like(shard), shard]) + + tuned = torch.full((4, HIDDEN), 3.0) + with ( + patch.object( + attn_residual.AttnResidual, "forward_sp_all_gather", return_value=None + ), + patch.object(comm_moves, "attn_tp_gather", all_gather), + ): + for gather in (None, lambda h: None): + got = self.final_read(holder, gather)(self.hidden[2:]) + torch.testing.assert_close(got[2:], every_row[2:], rtol=0, atol=0) + got = self.final_read(holder, lambda h: tuned)(self.hidden[2:]) + self.assertIs(got, tuned) + fused = torch.full((4, HIDDEN), 5.0) + with patch.object( + attn_residual.AttnResidual, + "forward_sp_all_gather", + return_value=(fused, None), + ) as kernel: + got = self.final_read(holder, lambda h: tuned)(self.hidden[2:]) + self.assertIs(got, fused) + self.assertEqual(kernel.call_args.kwargs["rows"], slice(2, 4)) + + def test_a_read_of_every_row_after_a_shard_write_is_refused(self): + holder = self.open() + layer = LAYER_LIST[2] + self.assertTrue(layer.writes_block) + layer.ops(holder).attn_readout.read(self.hidden[2:], layer.input_norm) + with self.assertRaisesRegex(RuntimeError, "stale"): + self.final_read(holder)(self.hidden) + with ( + patch.object( + attn_residual.AttnResidual, "forward_sp_all_gather", return_value=None + ), + patch.object( + comm_moves, + "attn_tp_gather", + lambda shard, owned=False: torch.cat([shard, shard]), + ), + ): + self.final_read(holder)(self.hidden[2:]) + + def test_the_fused_reads_are_offered_on_request(self): + def scatter_add(hidden_states, residual, forward_batch): + return None + + scatter = ReadoutFusion(SumGroup.ATTN_TP, scatter_add, scatters=True) + for fuses in (False, True): + with self.subTest(fuses_slice_collectives=fuses): + ops = AttnBankState( + AttnBank(), + LAYER_LIST[1].attn_proj, + LAYER_LIST[1].attn_score_norm, + LAYER_LIST[1].ffn_proj, + LAYER_LIST[1].ffn_score_norm, + ffn_input_fusions=(scatter,), + fuses_slice_collectives=fuses, + ).residual_ops() + gathers = ops.attn_readout.gathering_reads + fusions = ops.ffn_readout.completing_fusions + if not fuses: + self.assertEqual(gathers, ()) + self.assertEqual(fusions, (scatter,)) + continue + self.assertEqual(len(gathers), 1) + # The fused reduce-scatter and read first, then the + # reduce-scatter with the residual add alone. + self.assertEqual([f.reads for f in fusions], [True, False]) + self.assertTrue(fusions[0].scatters) + self.assertIs(fusions[1], scatter) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py new file mode 100644 index 0000000000..1429481936 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py @@ -0,0 +1,204 @@ +"""Kimi K3's own kernels for a layer's sums and gathers serve its +attention-residual bank only: the standard residual path keeps the +boundary's collectives, as the decoder did, even where those kernels run. +With SGLANG_K3_SP_ATTN_RES the bank and the stream stay on each rank's shard +across SP-MoE layers.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import test_declared_decoder_boundary as fixture +from torch import nn + +from sglang.srt.environ import envs +from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective +from sglang.srt.layers.layer_boundary import BatchVariant, layer_stack +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, keep_output +from sglang.srt.layers.layer_boundary.residual.attn_bank import AttnBank +from sglang.srt.models import kimi_k3 +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +CONFIG = SimpleNamespace( + is_moe=True, num_experts=8, first_k_dense_replace=1, moe_layer_freq=1 +) + + +def moe_layer(*, bank, sp_moe, all_reduce_fusion): + """A K3 latent MoE layer's attributes that its stage declarations read.""" + layer = kimi_k3.KimiK3DecoderLayer.__new__(kimi_k3.KimiK3DecoderLayer) + nn.Module.__init__(layer) + layer.use_attn_residuals = bank + layer.is_block_write_layer = False + layer._is_moe_layer = True + layer._sp_moe = sp_moe + layer.all_reduce_fusion = all_reduce_fusion + layer._ffn_writes_stream = bank + layer.mlp = SimpleNamespace(use_latent_moe=True, _ep_a2a=sp_moe) + layer.input_layernorm = fixture.Norm() + layer.post_attention_layernorm = fixture.Norm() + for name in ( + "self_attention_res_proj", + "self_attention_res_norm", + "mlp_res_proj", + "mlp_res_norm", + ): + setattr(layer, name, None) + return layer + + +def declare(layers, *, sp_moe, carries=False, config=CONFIG): + """Declare ``layers`` (index -> layer) in one layer stack, as if the + stack carried the bank's shards (``carries``).""" + bank = AttnBank() + with ( + patch.object(k3_sp_collective, "enabled", return_value=True), + patch.object(k3_ar_fusion, "enabled", return_value=True), + patch.object(kimi_k3, "_carries_bank_slices", return_value=carries), + envs.SGLANG_K3_SP_ATTN_RES.override(carries), + fixture.planning( + fixture.parallel_of(attn_dp=1, attn_tp=2), + a2a=sp_moe, + boundary_reduction="ar", + ), + layer_stack(), + ): + for idx, layer in layers.items(): + layer._declare_stages(config, idx, bank) + + +class TestKimiK3StageKernels(CustomTestCase): + def ffn_path(self, *, bank, sp_moe, all_reduce_fusion): + layer = moe_layer(bank=bank, sp_moe=sp_moe, all_reduce_fusion=all_reduce_fusion) + declare({1: layer}, sp_moe=sp_moe) + return layer.ffn_boundary.plan.paths[BatchVariant.ORDINARY] + + @staticmethod + def kernels(path): + step = path.entry.prepare.keywords["step"] + gather = getattr(path.output_move, "keywords", {}).get("gather") + fusions = tuple(f.run for f in step.keywords["read_fusions"]) + return step.func, fusions, gather + + def test_the_standard_path_keeps_the_boundary_collectives(self): + for sp_moe in (True, False): + with self.subTest(sp_moe=sp_moe): + entry, fusions, gather = self.kernels( + self.ffn_path(bank=False, sp_moe=sp_moe, all_reduce_fusion=True) + ) + self.assertIs( + entry, + comm_ops._attn_tp_reduce_scatter_update_read + if sp_moe + else comm_ops._reduce_update_read, + ) + self.assertEqual(fusions, ()) + self.assertIsNone(gather) + + def test_the_bank_takes_the_tuned_reduce_scatter_and_gather(self): + entry, fusions, gather = self.kernels( + self.ffn_path(bank=True, sp_moe=True, all_reduce_fusion=False) + ) + self.assertIs(entry, comm_ops._attn_tp_reduce_scatter_update_read) + self.assertEqual(fusions, (kimi_k3._k3_reduce_scatter_add,)) + self.assertIs(gather, k3_sp_collective.all_gather) + + def test_the_bank_takes_the_fused_all_reduce(self): + entry, fusions, gather = self.kernels( + self.ffn_path(bank=True, sp_moe=False, all_reduce_fusion=True) + ) + self.assertIs(entry, comm_ops._reduce_update_read) + self.assertEqual(fusions, (kimi_k3._k3_all_reduce_add,)) + self.assertIsNone(gather) + + +class TestKimiK3BankOnShards(CustomTestCase): + def stack(self, *, carries): + layers = { + idx: moe_layer(bank=True, sp_moe=True, all_reduce_fusion=False) + for idx in (1, 2, 3) + } + declare(layers, sp_moe=True, carries=carries) + return [ + ( + layer.attn_boundary.plan.paths[BatchVariant.ORDINARY], + layer.ffn_boundary.plan.paths[BatchVariant.ORDINARY], + ) + for layer in layers.values() + ] + + def test_consecutive_moe_layers_stay_on_their_shards(self): + for idx, (attn, ffn) in enumerate(self.stack(carries=True)): + with self.subTest(layer=idx + 1): + # The stack's last FFN too: the final read takes its shard. + self.assertIs(ffn.output_move, keep_output) + step = ffn.entry.prepare.keywords["step"] + self.assertIs(step.func, comm_ops._attn_tp_reduce_scatter_update_read) + self.assertEqual(step.keywords["scatters_residual"], idx == 0) + # The bank's fused reduce-scatter and read first, then K3's + # reduce-scatter with the residual add. + fusions = step.keywords["read_fusions"] + self.assertEqual([f.reads for f in fusions], [True, False]) + self.assertIs(fusions[1].run, kimi_k3._k3_reduce_scatter_add) + if idx == 0: + self.assertIsNone(attn.entry.input_move) + continue + # Read on the shard, then gathered: by the bank's kernel that + # does both, else K3's tuned all-gather, else the boundary's. + self.assertIs(attn.entry.input_move.func, attn_tp_gather_input) + self.assertIs( + attn.entry.input_move.keywords["gather"], + k3_sp_collective.all_gather, + ) + read = attn.entry.prepare.keywords["step"] + self.assertEqual(len(read.keywords["read_gathers"]), 1) + + def test_the_shard_mode_is_decided_at_construction(self): + # A rank holds only its own rows of the bank: no pipeline rank after + # it, no draft model capturing the target's hidden states, and no + # dense layer after an MoE layer. + cases = { + (None, 1, 1): True, + ("EAGLE", 1, 1): True, + (None, 2, 1): False, + ("EAGLE3", 1, 1): False, + ("DFLASH", 1, 1): False, + ("DSPARK", 1, 1): False, + (None, 1, 2): False, + } + for (algorithm, pp_size, moe_layer_freq), carries in cases.items(): + config = SimpleNamespace( + **{ + **vars(CONFIG), + "attn_res_block_size": 4, + "num_hidden_layers": 6, + "moe_layer_freq": moe_layer_freq, + } + ) + with ( + self.subTest( + algorithm=algorithm, + pp_size=pp_size, + moe_layer_freq=moe_layer_freq, + ), + patch.object(kimi_k3, "_shards_moe_rows", return_value=True), + patch.object(k3_sp_collective, "enabled", return_value=True), + envs.SGLANG_K3_SP_ATTN_RES.override(True), + patch.object( + kimi_k3, "get_parallel", lambda: SimpleNamespace(pp_size=pp_size) + ), + patch.object( + kimi_k3, + "get_spec", + lambda: SimpleNamespace(speculative_algorithm=algorithm), + ), + ): + self.assertEqual(kimi_k3._carries_bank_slices(config), carries) + + +if __name__ == "__main__": + unittest.main()