[Refactor] Run Kimi K3's fused collectives through stage boundaries (#42898)

This commit is contained in:
Cheng Wan
2026-10-09 01:27:23 -07:00
committed by GitHub
parent 2b469040d2
commit 305ad5503d
16 changed files with 1169 additions and 125 deletions
+14 -2
View File
@@ -460,6 +460,9 @@ class AttnResidual:
(num_tokens, block_num, hidden_size)
)
self.num_valid_blocks = 0
# Whether a write snapshotted only this rank's slice of the rows, which
# leaves the bank's other rows stale on this rank.
self.written_on_slice = False
if block_residual is not None: # inherited from the previous PP rank
self.num_valid_blocks = block_residual.size(1)
self.block_residual[:, : self.num_valid_blocks, :].copy_(block_residual)
@@ -472,7 +475,11 @@ class AttnResidual:
"""
bank = self.block_residual if rows is None else self.block_residual[rows]
bank[:, self.num_valid_blocks, :].copy_(prefix_sum)
self._count_write(rows)
def _count_write(self, rows: Optional[slice]) -> None:
self.num_valid_blocks += 1
self.written_on_slice |= rows is not None
def forward(
self,
@@ -488,6 +495,11 @@ class AttnResidual:
(the second return value) into the next bank row — fused into the
fast kernel (the row streams through its score pass anyway), a
standalone .write() copy on every other path."""
if rows is None and self.written_on_slice:
raise RuntimeError(
"a read of every row after a write of this rank's slice of "
"them would aggregate stale snapshots"
)
nvb = self.num_valid_blocks
# Layer 0 attention side: nothing banked yet
if nvb == 0:
@@ -533,7 +545,7 @@ class AttnResidual:
write_bank_row=fused_write,
)
if fused_write:
self.num_valid_blocks += 1 # row nvb written in-kernel
self._count_write(rows) # row nvb written in-kernel
elif write:
self.write(prefix, rows)
return normed, prefix
@@ -572,7 +584,7 @@ class AttnResidual:
if normed is None:
return None
if write:
self.num_valid_blocks += 1
self._count_write(rows)
return normed, prefix
def forward_sp_reduce_scatter(
@@ -34,6 +34,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
InputContract,
OutputContract,
ProducerReduction,
ReadoutFusion,
StageContract,
StageKind,
StagePath,
@@ -96,6 +97,7 @@ __all__ = [
"ExitDecision",
"FfnExit",
"FfnInputFusion",
"ReadoutFusion",
"DeferredFinalize",
"GatedResidualState",
"IHCState",
@@ -159,10 +159,14 @@ def get_attn_tp_context():
return ATTN_TP_CONTEXT
def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
def attn_tp_gather(tensor: torch.Tensor, *, owned: bool = False) -> torch.Tensor:
"""Gather this rank's attention-TP slice of the rows into all of them: in
the local DP buffer, or in a buffer of its own for an output that
outlives the next use of that buffer (``owned``)."""
parallel = get_parallel()
if batches_are_unpadded():
# MLP sync, which sizes the local DP buffer for a batch, does not run.
if owned or batches_are_unpadded():
# MLP sync, which sizes the local DP buffer for a batch, does not run
# for an unpadded one.
gathered = tensor.new_empty(
(tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1])
)
@@ -187,6 +187,7 @@ def bind_entry(
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
attn_input_adapter: Optional[Callable] = None,
attn_tp_gather: Optional[Callable] = None,
) -> EntryPath:
"""Bind the consumer half of an edge at construction time.
@@ -197,6 +198,8 @@ def bind_entry(
cp_moves: Context-parallel strategy operations for this edge, if needed.
enters_stack: Whether the read must initialize the stack's residual.
attn_input_adapter: Hands an attention its input once it is on its rows.
attn_tp_gather: The stage's implementation of a gather over attention
TP, tried before the default one.
Returns:
An EntryPath whose prepare accepts the actual update from the stream.
@@ -218,6 +221,11 @@ def bind_entry(
if edge.produced.update is None:
raise ValueError("an arrival must declare its update capabilities")
capabilities = (edge.produced.update.is_plain_add,)
gather_input = (
attn_tp_gather_input
if attn_tp_gather is None
else partial(attn_tp_gather_input, gather=attn_tp_gather)
)
paths = {
capability: _bind_entry_path(
edge,
@@ -227,6 +235,7 @@ def bind_entry(
cp_moves=cp_moves,
enters_stack=enters_stack,
attn_input_adapter=attn_input_adapter,
gather_input=gather_input,
)
for capability in capabilities
}
@@ -254,6 +263,7 @@ def _bind_entry_path(
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
attn_input_adapter: Optional[Callable] = None,
gather_input: Callable = attn_tp_gather_input,
) -> EntryPath:
"""The consumer's half of ``edge``, chosen from the edge's declarations and
the producer's update: what a value carries for a batch (a sum or a handoff
@@ -277,6 +287,8 @@ def _bind_entry_path(
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
enters_stack=enters_stack,
gather_input=gather_input,
arrives_written=edge.arrives_written,
)
completed_step = None
if edge.produced.always_partial:
@@ -343,7 +355,12 @@ def _bind_entry_path(
)
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove:
def bind_exit(
edge: EdgeContract,
*,
cp_moves: Optional[CpMoves] = None,
attn_tp_gather: Optional[Callable] = None,
) -> ExitMove:
"""Bind the producer half of a layer or branch exit.
Args:
@@ -351,6 +368,8 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit
must outlive the producer; pipeline handoffs require a plain add
or a residual already written by the producer.
cp_moves: Context-parallel return operations, when the edge needs them.
attn_tp_gather: The stage's implementation of a gather over attention
TP, tried before the default one.
Returns:
An ExitMove with fixed transport or a marker for batch-dependent DP
@@ -373,6 +392,20 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit
residual=edge.residual,
to=edge.residual_to,
cp_moves=cp_moves,
attn_tp_gather=attn_tp_gather,
)
def _read_fusions(read, owes, is_plain_add, *, scatters):
"""The kernels a read supplies that complete ``owes`` on the rows this
entry completes it on. They take the residual add themselves, so only a
plain one."""
if not is_plain_add:
return ()
return tuple(
f
for f in getattr(read, "completing_fusions", ())
if f.completes is owes and f.scatters == scatters
)
@@ -387,6 +420,8 @@ def _select_entry_step(
residual_joins_sum: bool,
cp_moves: Optional[CpMoves],
enters_stack: bool,
gather_input: Callable = attn_tp_gather_input,
arrives_written: bool = False,
) -> Tuple[Callable, Optional[Callable]]:
"""The steps into a stage for a value that is complete or owes the sum its
producer always leaves, the fused kernels they try first, and the move
@@ -447,14 +482,15 @@ def _select_entry_step(
or residual not in (produced.layout, need.layout)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
if owes is SumGroup.ATTN_TP:
step = partial(
_attn_tp_reduce_scatter_update_read,
read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True),
)
else:
step = _attn_tp_slice_update_read
return (
partial(
_attn_tp_reduce_scatter_update_read
if owes is SumGroup.ATTN_TP
else _attn_tp_slice_update_read,
scatters_residual=residual != residual_to,
read=read,
),
partial(step, scatters_residual=residual != residual_to, read=read),
None,
)
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}:
@@ -507,7 +543,7 @@ def _select_entry_step(
return (partial(cp_moves.gather, gather=on_chunk), None)
if gathered == {TokenAxis.ATTN_TP}:
# A complete input on each rank's slice, gathered over attention TP
# once it is read.
# once it is read, unless one of the read's own kernels gathers it.
if (
owes is not None
or residual != residual_to
@@ -520,8 +556,13 @@ def _select_entry_step(
pre_move=None,
enters_stack=enters_stack,
read=read,
read_gathers=(
getattr(read, "gathering_reads", ())
if is_plain_add or arrives_written
else ()
),
),
attn_tp_gather_input,
gather_input,
)
if (
residual_to != produced.layout
@@ -572,11 +613,13 @@ def _select_entry_step(
):
raise NotImplementedError(f"{produced=} {need=}")
fused = tuple(f for f in fusions if f.completes is owes)
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
return (
partial(
_reduce_update_read,
gathers_residual=gathers_residual,
fusions=tuple(f.run for f in fused),
read_fusions=read_fused,
group=owes,
read=read,
),
@@ -625,6 +668,7 @@ def _select_exit_move(
residual: Layout,
to: Layout,
cp_moves: Optional[CpMoves] = None,
attn_tp_gather: Optional[Callable] = None,
) -> ExitMove:
"""How the FFN output reaches the rows the layer hands on: by undoing the
attention-DP gather (the FFN exit and finish run that step), or else the
@@ -638,7 +682,13 @@ def _select_exit_move(
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP}:
# Each rank's slice back to the attention's rows: write the output
# into the residual, then gather over attention TP.
return ExitMove(partial(update_attn_tp_gather_output, update=update))
return ExitMove(
partial(
update_attn_tp_gather_output,
update=update,
gather=attn_tp_gather,
)
)
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if returned == {TokenAxis.ATTN_TP} and to in (residual, produced.layout):
@@ -47,6 +47,10 @@ from sglang.srt.layers.layer_boundary.layout import (
batches_are_unpadded,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.ops import (
attn_tp_gather_input,
update_attn_tp_gather_output,
)
from sglang.srt.layers.layer_boundary.prepare import (
_attn_input_default,
_attn_input_scattered,
@@ -186,6 +190,7 @@ class StagePlan:
finishes_directly=False,
qkv_latent_func=None,
fusions=None,
attn_tp_gather=None,
):
self.norm = norm
self.edges = dict(variants)
@@ -229,11 +234,16 @@ class StagePlan:
cp_moves=edges.cp_moves,
enters_stack=enters_stack,
attn_input_adapter=edges.attn_input_adapter,
attn_tp_gather=attn_tp_gather,
)
out = (
ExitMove()
if finishes_directly
else bind_exit(edges.outgoing, cp_moves=edges.cp_moves)
else bind_exit(
edges.outgoing,
cp_moves=edges.cp_moves,
attn_tp_gather=attn_tp_gather,
)
)
self.paths[variant] = StagePath(
entry=entry,
@@ -243,6 +253,15 @@ class StagePlan:
returns_over_dp=out.returns_over_dp,
writes_at_handoff=writes_at_handoff,
)
if attn_tp_gather is not None and not any(
getattr(path.output_move, "func", None) is update_attn_tp_gather_output
or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
for path in self.paths.values()
):
raise ValueError(
"an attention-TP gather is declared, but no batch variant "
"gathers over attention TP at this stage's boundaries"
)
@property
def incoming_residual_rows(self):
@@ -334,6 +353,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
is_branch=declaration.prepared_from is not None,
terminal=declaration.terminal,
writes_at_handoff=declaration.writes_at_handoff,
attn_tp_gather=declaration.attn_tp_gather,
finishes_directly=declaration.kind is StageKind.ATTENTION
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
**options,
@@ -14,7 +14,7 @@
"""Immutable stage declarations and bound input/output paths."""
from enum import Enum, auto
from typing import Callable, FrozenSet, Optional, Tuple
from typing import Callable, FrozenSet, Optional, Tuple, Union
import msgspec
import torch
@@ -45,6 +45,10 @@ class ExitRows(Enum):
ATTENTION = auto()
TBO_SPLIT = auto()
# The rows the FFN ran on, also at the stack's end: an FFN on this rank's
# attention-TP slice leaves its output there, and the model's final read
# must read that slice and gather it.
SLICE = auto()
class BatchVariant(Enum):
@@ -139,6 +143,8 @@ class EdgeContract(msgspec.Struct, frozen=True):
arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for
arriving contributions. Empty means use produced.update's capability.
The actual update object travels with the residual stream.
arrives_written: Whether the producer applies its update at its exit,
so the stream arrives written with no residual add pending.
"""
produced: OutputContract
@@ -150,6 +156,7 @@ class EdgeContract(msgspec.Struct, frozen=True):
residual_joins_sum: bool = False
# Capabilities allowed to arrive from another layer, not its update object.
arriving_plain_add: Tuple[bool, ...] = ()
arrives_written: bool = False
class FfnInputFusion(msgspec.Struct, frozen=True):
@@ -168,6 +175,28 @@ class FfnInputFusion(msgspec.Struct, frozen=True):
preserves_residual: Optional[Callable] = None
class ReadoutFusion(msgspec.Struct, frozen=True):
"""A kernel a stage's read supplies (its ``completing_fusions``) that
completes the sum its input owes together with the residual add, for a
read that is not the residual's plain norm and so takes no FfnInputFusion.
``run(hidden_states, residual, forward_batch)`` returns the written stream,
the completed sum plus the residual, which the read then reads; or None
when it does not take the batch, before touching its inputs or starting a
collective."""
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]]
# Whether it completes the sum onto this rank's attention-TP slice of the
# rows (a reduce-scatter, given the whole residual or its slice) rather
# than on every row.
scatters: bool = False
# Whether it also does the read: run(hidden_states, residual,
# forward_batch, norm) then returns the read's (input, residual).
reads: bool = False
class CpMoves(msgspec.Struct, frozen=True):
"""How a CP extend's rows reach an FFN that needs all of them and come back,
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
@@ -7,7 +7,7 @@ import os
import sys
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Mapping, Optional
from typing import Callable, Mapping, Optional
import msgspec
@@ -260,6 +260,11 @@ class StageDeclaration:
output_complete: Whether the FFN's compute completes its own output
sum, so the exit owes none. For an FFN whose output sum is fused
with a reduction its computation needs anyway.
attn_tp_gather: Optional implementation of the gather over attention
TP this stage's boundaries run: given this rank's contiguous slice
of the rows, it returns them all, in rank order, or None to leave
the gather to the boundary. Called on every batch, so it must be
CUDA-graph safe.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
prepared_from: Declaration whose already-read input a branch reuses.
@@ -281,6 +286,7 @@ class StageDeclaration:
exit_rows: Optional[ExitRows] = None
writes_at_handoff: bool = False
output_complete: bool = False
attn_tp_gather: Optional[Callable] = None
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -325,6 +331,7 @@ def declare_attn(
gathers_attn_tp_input=True,
output_transform=None,
tp_group=SumGroup.ATTN_TP,
attn_tp_gather=None,
):
"""Declare attention or a mixer; construct its executable boundary later.
@@ -339,6 +346,8 @@ def declare_attn(
stage's input runs it, so no fused add + norm takes that input.
Requires ALWAYS_PARTIAL.
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
attn_tp_gather: Implementation of this stage's gathers over attention
TP, tried before the boundary's own (see StageDeclaration).
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -356,6 +365,7 @@ def declare_attn(
reduction=reduction,
gathers_attn_tp_input=gathers_attn_tp_input,
tp_group=tp_group,
attn_tp_gather=attn_tp_gather,
)
@@ -369,6 +379,7 @@ def declare_ffn(
dense_tp_size=None,
exit_rows=None,
output_complete=False,
attn_tp_gather=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -385,6 +396,8 @@ def declare_ffn(
the adjacent FFN kinds and TBO configuration.
output_complete: Whether the compute completes its own output sum,
fused with a reduction it needs anyway; the exit then owes none.
attn_tp_gather: Implementation of this stage's gathers over attention
TP, tried before the boundary's own (see StageDeclaration).
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -398,6 +411,7 @@ def declare_ffn(
dense_tp_size=dense_tp_size,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
output_complete=output_complete,
attn_tp_gather=attn_tp_gather,
)
@@ -423,8 +437,13 @@ def _resolve_stage(stage, variant, following=None):
dense_tp_size=stage.dense_tp_size,
output_complete=stage.output_complete,
)
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
exit_rows = resolve_exit_rows(stage.exit_rows)
if exit_rows is ExitRows.ATTENTION:
returned = attention
elif exit_rows is ExitRows.SLICE:
# The residual's rows during the FFN: this rank's slice for an FFN
# on its own rows, else the attention's.
returned = residual
return declaration, residual, returned
sp = variant is BatchVariant.SEQUENCE_PARALLEL
scattered = variant is BatchVariant.INPUT_SCATTERED
@@ -489,6 +508,7 @@ def _connect(producer, consumer, *, residual_from=None):
exits, entries = {}, {}
for variant in _active_variants():
_, attention, local, _ = _row_layouts(variant)
written = False
if before is None:
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
owes = variant is BatchVariant.INPUT_SCATTERED
@@ -548,6 +568,7 @@ def _connect(producer, consumer, *, residual_from=None):
update=None,
)
residual, capabilities = returned, (before.update.is_plain_add,)
written = before.update.applied_at_exit
if after is None:
continue
decl, during, _ = _resolve_stage(after, variant)
@@ -579,6 +600,7 @@ def _connect(producer, consumer, *, residual_from=None):
during,
residual_joins_sum=joins,
arriving_plain_add=capabilities,
arrives_written=written,
)
entries[variant] = edge
if (
+31 -2
View File
@@ -91,10 +91,27 @@ def tp_slice(hidden_states, residual):
return hidden_states, residual
class GatheredInput:
"""A stage input its read already gathered over attention TP, which the
entry's gather then passes on as it is."""
__slots__ = ("value",)
def __init__(self, value: torch.Tensor):
self.value = value
def attn_tp_gather_input(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
gather: Optional[Callable] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""Gather a stage input from this rank's attention-TP slice of the rows,
trying the stage's ``gather`` first for a single tensor (see
attn_tp_gather_with); an input its read already gathered (GatheredInput)
passes through."""
if isinstance(hidden_states, GatheredInput):
return hidden_states.value
parallel = get_parallel()
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
@@ -118,7 +135,7 @@ def attn_tp_gather_input(
gathered_hidden_states.append(output)
return tuple(gathered_hidden_states)
return attn_tp_gather(hidden_states)
return attn_tp_gather_with(hidden_states, gather)
def attn_tp_reduce_scatter(
@@ -328,10 +345,22 @@ def update_attn_tp_gather_output(
residual: torch.Tensor,
forward_batch: ForwardBatch,
update: ResidualUpdate = PLAIN_ADD,
gather: Optional[Callable] = None,
**kwargs,
):
"""Write the output into the residual on this rank's slice, then gather it
over attention TP (see attn_tp_gather_with)."""
hidden_states = update.update(hidden_states, residual)
return attn_tp_gather(hidden_states), None
return attn_tp_gather_with(hidden_states, gather), None
def attn_tp_gather_with(
hidden_states: torch.Tensor, gather: Optional[Callable]
) -> torch.Tensor:
"""Gather this rank's attention-TP slice of the rows into all of them: in
the stage's own ``gather`` when it takes the batch, else the boundary's."""
gathered = gather(hidden_states) if gather is not None else None
return attn_tp_gather(hidden_states) if gathered is None else gathered
def residual_slice_output(
@@ -40,6 +40,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
get_attn_tp_context,
tp_gather,
)
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
_cp_shard_token_rows,
@@ -66,6 +67,7 @@ if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
from sglang.srt.layers.layer_boundary.ops import (
GatheredInput,
attn_tp_all_reduce,
attn_tp_reduce_scatter,
dp_gather,
@@ -74,6 +76,25 @@ from sglang.srt.layers.layer_boundary.ops import (
)
def _run_read_fusions(
read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call
):
"""The first of the read's own kernels (ReadoutFusion) that takes the
batch: one that completes the sum with the residual add hands the written
stream to ``read``; one that also reads returns the read. None when every
kernel declines."""
for fused in read_fusions:
if fused.reads:
result = fused.run(hidden_states, residual, forward_batch, norm)
if result is not None:
return result
continue
stream = fused.run(hidden_states, residual, forward_batch)
if stream is not None:
return read.read(stream, norm, **call)
return None
def _reduce_update_read(
hidden_states: torch.Tensor,
residual: torch.Tensor,
@@ -83,6 +104,7 @@ def _reduce_update_read(
cache=None,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
read_fusions: Tuple[ReadoutFusion, ...] = (),
group: SumGroup = SumGroup.ATTN_TP,
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
@@ -90,11 +112,24 @@ def _reduce_update_read(
post_residual_addition: Optional[torch.Tensor] = None,
):
"""Complete the sum the input owes over ``group`` on the rows it is on,
unless one of ``fusions`` does it with the residual add and the norm, then
write it into the residual and read the input (an attention's read takes
unless one of the read's ``read_fusions`` does it with the residual add,
or one of ``fusions`` with the residual add and the norm, then write it
into the residual and read the input (an attention's read takes
``quant_format`` and ``post_residual_addition``)."""
if gathers_residual:
residual = update.gather_residual_attn_tp(residual)
result = _run_read_fusions(
read_fusions,
hidden_states,
residual,
forward_batch,
norm,
read=read,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
if result is not None:
return result
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
@@ -315,6 +350,7 @@ def _update_read(
pre_move: Optional[Callable],
enters_stack: bool,
read: ResidualReadout,
read_gathers: Tuple[Callable, ...] = (),
update: ResidualUpdate = PLAIN_ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
@@ -322,12 +358,19 @@ def _update_read(
"""Complete what the input owes by construction (``pre_move``), then
write the previous stage's output into the residual and read the stage's
input with ``norm``. The layer stack's first stage (``enters_stack``)
starts its residual from its input."""
starts its residual from its input. The first of ``read_gathers`` that
takes the batch reads and gathers the input itself; its result is wrapped
in GatheredInput so the entry's gather passes it through."""
enters = residual is None and enters_stack
if pre_move is not None:
hidden_states, residual = pre_move(hidden_states, residual)
if enters:
hidden_states, residual = read.init_residual(hidden_states), None
for fused in read_gathers:
result = fused(hidden_states, residual, norm)
if result is not None:
gathered, residual = result
return GatheredInput(gathered), residual
if residual is None:
# The previous layer already wrote its output into the residual.
return read.read(hidden_states, norm, quant_format)
@@ -349,9 +392,19 @@ def _attn_tp_reduce_scatter_update_read(
*,
cache=None,
scatters_residual: bool,
read_fusions: Tuple[ReadoutFusion, ...] = (),
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
):
"""Complete the attention-TP sum onto this rank's slice, slice the residual
the same way, then write the output into it and read the input; the first
of the read's ``read_fusions`` that takes the batch does all of this
instead."""
result = _run_read_fusions(
read_fusions, hidden_states, residual, forward_batch, norm, read=read
)
if result is not None:
return result
hidden_states = attn_tp_reduce_scatter(hidden_states)
if scatters_residual and residual is not None:
residual = update.slice_residual_attn_tp(residual)
@@ -70,6 +70,13 @@ class ResidualReadout(Protocol):
is_plain_norm: Read is normalization/optional quantization without changing
the residual, allowing compatible fused add+norm implementations.
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
completing_fusions: Optional. ReadoutFusion kernels that complete the
sum the input owes with the residual add ahead of this read, tried
before the boundary's own all-reduce or reduce-scatter.
gathering_reads: Optional. Callables ``(hidden_states, residual,
norm)`` that read the input on this rank's attention-TP slice and
gather it over attention TP, returning ``(input, residual)``, or
None when they do not take the batch.
init_residual initializes the stack residual. read consumes an
already-written residual; update_and_read first applies the actual
@@ -14,11 +14,14 @@
"""A residual read that aggregates a bank of snapshots of itself."""
from dataclasses import dataclass
from typing import Optional
from typing import Optional, Tuple
import torch
from sglang.srt.layers.attn_residual import AttnResidual
from sglang.srt.layers.layer_boundary import ops
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
from sglang.srt.runtime_context import get_parallel
@@ -87,6 +90,12 @@ class AttnBankState:
ffn_score_proj: torch.nn.Module
ffn_score_norm: torch.nn.Module
writes_block: bool = False
# Kernels that complete the attention output's sum with the pending add
# ahead of the FFN read.
ffn_input_fusions: Tuple[ReadoutFusion, ...] = ()
# A read on this rank's attention-TP slice first tries the bank's kernels
# that also run the collective around it.
fuses_slice_collectives: bool = False
def _aggregate(
self, contribution, residual, norm, *, score_proj, score_norm, write
@@ -131,6 +140,42 @@ class AttnBankState:
write=False,
)
def reduce_scatter_and_read_ffn_input(
self, hidden_states, residual, forward_batch, norm
):
parallel = get_parallel()
rows, rest = divmod(hidden_states.shape[0], parallel.attn_tp_size)
if rows == 0 or rest:
return None
start = parallel.attn_tp_rank * rows
return self.bank.require().forward_sp_reduce_scatter(
hidden_states,
residual,
self.ffn_score_proj,
self.ffn_score_norm,
norm,
rows=slice(start, start + rows),
)
def read_and_gather_attn_input(self, contribution, residual, norm):
bank = self.bank.require()
rows = _bank_rows(bank, contribution)
if rows is None:
return None
read = bank.forward_sp_all_gather(
contribution,
residual,
self.attn_score_proj,
self.attn_score_norm,
norm,
rows=rows,
write=self.writes_block,
)
if read is None:
return None
normed, residual = read
return normed, None if self.writes_block else residual
def residual_ops(self) -> LayerResidualOps:
return LayerResidualOps(
attn_readout=_AttnReadout(self),
@@ -171,6 +216,13 @@ class _AttnReadout(_BankReadout):
"""The attention input: the bank aggregation and this layer's input norm.
A write layer snapshots the residual this read forms."""
@property
def gathering_reads(self):
state = self.state
return (
(state.read_and_gather_attn_input,) if state.fuses_slice_collectives else ()
)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
self._reject(None, quant_format)
if post_residual_addition is not None:
@@ -188,6 +240,19 @@ class _FfnReadout(_BankReadout):
"""The FFN input: the attention output's add folded into the bank
aggregation, and this layer's post-attention norm."""
@property
def completing_fusions(self) -> Tuple[ReadoutFusion, ...]:
state = self.state
if not state.fuses_slice_collectives:
return state.ffn_input_fusions
reads = ReadoutFusion(
SumGroup.ATTN_TP,
state.reduce_scatter_and_read_ffn_input,
scatters=True,
reads=True,
)
return (reads, *state.ffn_input_fusions)
def read(self, residual, norm, quant_format="", post_residual_addition=None):
# The attention read of a write layer leaves the residual empty, so the
# attention output alone is the head.
@@ -202,22 +267,40 @@ class _FfnReadout(_BankReadout):
class AttnBankOutputRead:
"""The layer stack's terminal read: the bank aggregation with the output
side's scoring parameters, then the final norm. A callable final norm for
`residual_batch.final_norm`."""
`residual_batch.final_norm`.
def __init__(self, bank: AttnBank, score_proj, score_norm, norm):
A stack that ends on this rank's attention-TP slice of the rows is read
there, against the bank rows this rank wrote, and what it read is
gathered: by the bank's kernel that does both when it takes the batch,
else by ``attn_tp_gather`` when that does, else over the attention-TP
group."""
def __init__(
self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None
):
self.bank = bank
self.score_proj = score_proj
self.score_norm = score_norm
self.norm = norm
self.attn_tp_gather = attn_tp_gather
def __call__(self, hidden_states, residual=None):
bank = self.bank.require()
normed, head = bank.forward(
hidden_states,
residual,
self.score_proj,
self.score_norm,
self.norm,
rows=_bank_rows(bank, hidden_states),
rows = _bank_rows(bank, hidden_states)
read_args = (hidden_states, residual, self.score_proj, self.score_norm)
read = (
None
if rows is None
else bank.forward_sp_all_gather(*read_args, self.norm, rows=rows)
)
if read is None:
read = bank.forward(*read_args, self.norm, rows=rows)
if rows is not None:
read = (self._gather(read[0]), read[1])
normed, head = read
return normed if residual is None else (normed, head)
def _gather(self, normed):
gathered = None if self.attn_tp_gather is None else self.attn_tp_gather(normed)
# The output outlives the layer stack.
return ops.attn_tp_gather(normed, owned=True) if gathered is None else gathered
+185 -89
View File
@@ -51,6 +51,8 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.layer_boundary import (
ExitRows,
ReadoutFusion,
SumGroup,
append_stages,
declare_attn,
declare_ffn,
@@ -142,7 +144,9 @@ from sglang.srt.runtime_context import (
get_exec,
get_parallel,
get_platform,
get_spec,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils import is_hip, is_npu, make_pp_layers
from sglang.srt.utils.common import (
BumpAllocator,
@@ -428,6 +432,48 @@ def _o_proj_takes_output(o_proj: RowParallelLinear) -> bool:
return getattr(o_proj.quant_method, "apply_into", None) is not None
def _k3_all_reduce_add(hidden_states, residual, forward_batch):
"""o_proj's attention-TP sum with the pending residual add in K3's fused
all-reduce, for an output o_proj wrote into its multicast buffer."""
if k3_ar_fusion.find_mc_ptr(hidden_states) is None:
return None
return k3_ar_fusion.all_reduce(hidden_states, residual)
def _k3_reduce_scatter_add(hidden_states, residual, forward_batch):
"""o_proj's attention-TP sum onto this rank's slice with the pending
residual add, in K3's tuned reduce-scatter when it takes the batch."""
return k3_sp_collective.reduce_scatter_res(hidden_states, residual)
def _route_sp_o_proj_output(o_proj: RowParallelLinear) -> None:
"""o_proj under SP-MoE."""
# o_proj emits TP-partial sums, which the stage boundary completes.
o_proj.reduce_results = False
if k3_sp_collective.enabled():
# The table selects NVLS pull RS for larger token buckets.
# Only those o_proj outputs come from the persistent symmetric
# buffer; small push RS keeps the regular graph allocator.
_sp_inner_o_proj_forward = o_proj.forward
def _sp_o_proj_forward(x, *args, **kwargs):
output_rows = x.shape[0]
if k3_sp_collective.requires_symmetric_rs(
output_rows, x.device, x.element_size()
):
output = k3_sp_collective.get_o_proj_output_buffer(
output_rows, x.dtype, o_proj.output_size
)
result = _sp_inner_o_proj_forward(
x, *args, output_tensor=output[: x.shape[0]], **kwargs
)
k3_sp_collective.register_o_proj_output(result[0], output)
return result
return _sp_inner_o_proj_forward(x, *args, **kwargs)
o_proj.forward = _sp_o_proj_forward
def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Tensor:
"""Symmetric storage for o_proj's TP-partial output; the fused attention
all-reduce reduces it in place."""
@@ -1546,6 +1592,7 @@ class KimiK3DeltaAttention(nn.Module):
all_reduce_fusion: bool = False,
bfa_alt_stream: Optional[torch.cuda.Stream] = None,
reduce_results: bool = True,
sp_moe: bool = False,
**kwargs,
) -> None:
super().__init__()
@@ -1818,7 +1865,7 @@ class KimiK3DeltaAttention(nn.Module):
# the fused AR reduces o_proj's output in place out of a symmetric
# buffer, which needs the GEMM to write into caller-owned storage
self.all_reduce_fusion = False
self.o_proj.reduce_results = True
self.o_proj.reduce_results = reduce_results
self.o_proj.use_dp_attention_reduce = True
conv_weights = self.qkv_conv1d.weight.squeeze(1)
bias = self.qkv_conv1d.bias
@@ -1841,6 +1888,8 @@ class KimiK3DeltaAttention(nn.Module):
# Set by _prepare_fused_decode() once weights are loaded.
self._kda_fused_decode_ready = False
self._kda_hip_fused_decode_ready = False
if sp_moe:
_route_sp_o_proj_output(self.o_proj)
def forward_qkvbfg(self, hidden_states: torch.Tensor):
qkv, _ = self.qkv_proj(hidden_states)
@@ -2200,6 +2249,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
alt_stream: Optional[torch.cuda.Stream] = None,
gate_alt_stream: Optional[torch.cuda.Stream] = None,
reduce_results: bool = True,
sp_moe: bool = False,
) -> None:
# ModelSlim can quantize K3 latent projections while still storing
# MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V.
@@ -2276,7 +2326,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
# the fused AR reduces o_proj's output in place out of a symmetric
# buffer, which needs the GEMM to write into caller-owned storage
self.all_reduce_fusion = False
self.o_proj.reduce_results = True
self.o_proj.reduce_results = reduce_results
self.o_proj.use_dp_attention_reduce = True
if self.all_reduce_fusion:
# Hand the GEMM a slice of the persistent symmetric buffer
@@ -2350,6 +2400,8 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
return _orig_o_proj_forward(x, *args, **kwargs)
self.o_proj.forward = _gated_o_proj_forward
if sp_moe:
_route_sp_o_proj_output(self.o_proj)
@staticmethod
def _split_kv_b_weight_loader(param, loaded_weight) -> None:
@@ -2445,17 +2497,34 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
)
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
"""Whether the layers build stage boundaries, which is the same for every
layer of a stack. The layer's own communication still runs for: an
attention-residual bank whose o_proj all-reduce is fused with the pending
add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned
SP collectives, which the sharded carry also needs."""
if _fuses_attn_all_reduce(config):
def _carries_bank_slices(config: KimiLinearConfig) -> bool:
"""Whether consecutive SP-MoE layers keep the stream and the
attention-residual bank on each rank's attention-TP shard of the rows
(SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then
gathers what it read. A rank then holds only its own rows of the bank, so
neither a later pipeline rank nor a draft model's capture of the target's
hidden states can take them; it is decided at construction. Nor can a
dense layer after an SP-MoE one, whose FFN reads every row of the bank:
every layer from the first MoE layer on must be one."""
if not (
_shards_moe_rows()
and config.attn_res_block_size is not None
and k3_sp_collective.enabled()
and envs.SGLANG_K3_SP_ATTN_RES.get()
and get_parallel().pp_size == 1
):
return False
if not _shards_moe_rows():
return True
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
moe = [_is_moe_layer(config, idx) for idx in range(config.num_hidden_layers)]
if True not in moe or not all(moe[moe.index(True) :]):
return False
spec = SpeculativeAlgorithm.from_string(get_spec().speculative_algorithm)
return not (spec.is_eagle3() or spec.is_dflash_family())
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
"""Whether the layers build stage boundaries, which every configuration
does."""
return True
class KimiK3DecoderLayer(nn.Module):
@@ -2500,6 +2569,7 @@ class KimiK3DecoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
all_reduce_fusion=self.all_reduce_fusion,
reduce_results=not self._stage_boundaries,
sp_moe=self._sp_moe,
# Shared with the MLA gate stream: KDA and MLA layers never
# run concurrently within one forward, so the stream is free.
bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None),
@@ -2512,6 +2582,7 @@ class KimiK3DecoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
all_reduce_fusion=self.all_reduce_fusion,
reduce_results=not self._stage_boundaries,
sp_moe=self._sp_moe,
alt_stream=alt_streams[1] if alt_streams is not None else None,
gate_alt_stream=alt_streams[2] if alt_streams is not None else None,
)
@@ -2571,35 +2642,6 @@ class KimiK3DecoderLayer(nn.Module):
prefix=f"{prefix}.mlp_res_proj",
)
if self._sp_moe:
# o_proj emits TP-partial sums; _finish_attn_reduce completes the
# reduction (RS on the clean attn-res path, AR on fallbacks).
o_proj = getattr(self.self_attn, "o_proj", None)
assert o_proj is not None, "SP-MoE requires attention exposing o_proj"
o_proj.reduce_results = False
if k3_sp_collective.enabled():
# The table selects NVLS pull RS for larger token buckets.
# Only those o_proj outputs come from the persistent symmetric
# buffer; small push RS keeps the regular graph allocator.
_sp_inner_o_proj_forward = o_proj.forward
def _sp_o_proj_forward(x, *args, **kwargs):
output_rows = x.shape[0]
if k3_sp_collective.requires_symmetric_rs(
output_rows, x.device, x.element_size()
):
output = k3_sp_collective.get_o_proj_output_buffer(
output_rows, x.dtype, o_proj.output_size
)
result = _sp_inner_o_proj_forward(
x, *args, output_tensor=output[: x.shape[0]], **kwargs
)
k3_sp_collective.register_o_proj_output(result[0], output)
return result
return _sp_inner_o_proj_forward(x, *args, **kwargs)
o_proj.forward = _sp_o_proj_forward
# A latent MoE on this rank's rows adds the residual in its tail add and
# writes the next stream itself; one gathered over attention DP returns
# its output to this rank's tokens first.
@@ -2611,56 +2653,98 @@ class KimiK3DecoderLayer(nn.Module):
and not (self._dp_attention and not self.mlp._ep_a2a)
)
if self._stage_boundaries:
# Under attention DP the FFN input is read on this rank's rows
# once the attention's sum is complete, then gathered.
attn_ops = {}
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
if self.use_attn_residuals:
bank_ops = AttnBankState(
attn_bank,
self.self_attention_res_proj,
self.self_attention_res_norm,
self.mlp_res_proj,
self.mlp_res_norm,
writes_block=self.is_block_write_layer,
).residual_ops()
attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update)
ffn_ops = dict(
read=bank_ops.ffn_readout,
update=(
REPLACE_AT_EXIT
if self._ffn_writes_stream
else bank_ops.ffn_update
),
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(**attn_ops), self.input_layernorm),
(
declare_ffn(
**ffn_ops,
sparse=self._is_moe_layer,
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
# The TP width the dense MLP is built with.
dense_tp_size=(
None
if self._is_moe_layer
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
),
# SP-MoE runs on this rank's attention-TP shard of the
# rows; on the bank path, whose reads write the bank on
# every row, its output returns to all of them.
exit_rows=(
ExitRows.ATTENTION
if self._sp_moe and self.use_attn_residuals
else None
),
# A latent MoE completes its output sum together with
# the latent reduction its norm needs.
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
),
self.post_attention_layernorm,
self._declare_stages(config, layer_idx, attn_bank)
def _declare_stages(self, config, layer_idx, attn_bank):
"""Declare this layer's attention and FFN stages: the reads and updates
around them, the rows the FFN takes and leaves, and the kernels K3
supplies for the bank path."""
# Under attention DP the FFN input is read on this rank's rows
# once the attention's sum is complete, then gathered.
attn_ops = {}
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
carries = _carries_bank_slices(config)
bank_sp_moe = self._sp_moe and self.use_attn_residuals
tuned_gather = (
k3_sp_collective.all_gather if k3_sp_collective.enabled() else None
)
if self.use_attn_residuals:
bank_ops = AttnBankState(
attn_bank,
self.self_attention_res_proj,
self.self_attention_res_norm,
self.mlp_res_proj,
self.mlp_res_norm,
writes_block=self.is_block_write_layer,
ffn_input_fusions=self._ffn_input_fusions(),
fuses_slice_collectives=self._sp_moe
and k3_sp_collective.enabled()
and envs.SGLANG_K3_SP_ATTN_RES.get(),
).residual_ops()
attn_ops = dict(
read=bank_ops.attn_readout,
update=bank_ops.attn_update,
# An input read on the shard an SP-MoE layer left is gathered
# in K3's tuned all-gather when it takes the batch.
attn_tp_gather=(
tuned_gather
if carries and _is_moe_layer(config, layer_idx - 1)
else None
),
)
ffn_ops = dict(
read=bank_ops.ffn_readout,
update=(
REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update
),
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(**attn_ops), self.input_layernorm),
(
declare_ffn(
**ffn_ops,
sparse=self._is_moe_layer,
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
# The TP width the dense MLP is built with.
dense_tp_size=(
None
if self._is_moe_layer
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
),
# SP-MoE runs on this rank's attention-TP shard of the
# rows; on the bank path, whose reads write the bank on
# every row, its output returns to all of them, unless
# the bank stays on the shard, read there to the end.
exit_rows=(
(ExitRows.SLICE if carries else ExitRows.ATTENTION)
if bank_sp_moe
else None
),
# A latent MoE completes its output sum together with
# the latent reduction its norm needs.
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
# The bank path's gather back to every row, in K3's
# tuned all-gather when it takes the batch.
attn_tp_gather=(
tuned_gather if bank_sp_moe and not carries else None
),
),
self.post_attention_layernorm,
),
)
def _ffn_input_fusions(self):
"""K3's kernels that complete the attention output's sum with the
pending residual add ahead of the bank's FFN read: the fused
all-reduce on every row, or the tuned reduce-scatter onto this rank's
attention-TP shard (SP-MoE)."""
if self.all_reduce_fusion:
return (ReadoutFusion(SumGroup.ATTN_TP, _k3_all_reduce_add),)
if self._sp_moe and k3_sp_collective.enabled():
return (
ReadoutFusion(SumGroup.ATTN_TP, _k3_reduce_scatter_add, scatters=True),
)
return ()
def _finish_attn_reduce(
self,
@@ -2949,6 +3033,7 @@ class KimiK3LinearModel(nn.Module):
self.dspark_layers_to_capture: Optional[list[int]] = None
self._dp_attention = is_dp_attention_enabled()
self._stage_boundaries = _uses_stage_boundaries(config)
self.carries_bank_slices = _carries_bank_slices(config)
if self.pp_group.is_first_rank:
embedding_quant_config = (
@@ -3016,6 +3101,11 @@ class KimiK3LinearModel(nn.Module):
self.output_attn_res_proj,
self.output_attn_res_norm,
self.norm,
attn_tp_gather=(
k3_sp_collective.all_gather
if self.carries_bank_slices
else None
),
)
)
else:
@@ -3359,6 +3449,12 @@ class KimiK3LinearForCausalLM(nn.Module):
raise ValueError(
"DSPARK requires explicit layer_ids for aux hidden capture."
)
if self.model.carries_bank_slices:
raise RuntimeError(
"the model keeps each rank's shard of the rows across SP-MoE "
"layers (SGLANG_K3_SP_ATTN_RES), which leaves no layer output "
"to capture whole"
)
self.capture_aux_hidden_states = True
self.model.dspark_layers_to_capture = list(layer_ids)