[Refactor] Run Kimi K3's fused collectives through stage boundaries (#42898)
This commit is contained in:
@@ -460,6 +460,9 @@ class AttnResidual:
|
||||
(num_tokens, block_num, hidden_size)
|
||||
)
|
||||
self.num_valid_blocks = 0
|
||||
# Whether a write snapshotted only this rank's slice of the rows, which
|
||||
# leaves the bank's other rows stale on this rank.
|
||||
self.written_on_slice = False
|
||||
if block_residual is not None: # inherited from the previous PP rank
|
||||
self.num_valid_blocks = block_residual.size(1)
|
||||
self.block_residual[:, : self.num_valid_blocks, :].copy_(block_residual)
|
||||
@@ -472,7 +475,11 @@ class AttnResidual:
|
||||
"""
|
||||
bank = self.block_residual if rows is None else self.block_residual[rows]
|
||||
bank[:, self.num_valid_blocks, :].copy_(prefix_sum)
|
||||
self._count_write(rows)
|
||||
|
||||
def _count_write(self, rows: Optional[slice]) -> None:
|
||||
self.num_valid_blocks += 1
|
||||
self.written_on_slice |= rows is not None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -488,6 +495,11 @@ class AttnResidual:
|
||||
(the second return value) into the next bank row — fused into the
|
||||
fast kernel (the row streams through its score pass anyway), a
|
||||
standalone .write() copy on every other path."""
|
||||
if rows is None and self.written_on_slice:
|
||||
raise RuntimeError(
|
||||
"a read of every row after a write of this rank's slice of "
|
||||
"them would aggregate stale snapshots"
|
||||
)
|
||||
nvb = self.num_valid_blocks
|
||||
# Layer 0 attention side: nothing banked yet
|
||||
if nvb == 0:
|
||||
@@ -533,7 +545,7 @@ class AttnResidual:
|
||||
write_bank_row=fused_write,
|
||||
)
|
||||
if fused_write:
|
||||
self.num_valid_blocks += 1 # row nvb written in-kernel
|
||||
self._count_write(rows) # row nvb written in-kernel
|
||||
elif write:
|
||||
self.write(prefix, rows)
|
||||
return normed, prefix
|
||||
@@ -572,7 +584,7 @@ class AttnResidual:
|
||||
if normed is None:
|
||||
return None
|
||||
if write:
|
||||
self.num_valid_blocks += 1
|
||||
self._count_write(rows)
|
||||
return normed, prefix
|
||||
|
||||
def forward_sp_reduce_scatter(
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
InputContract,
|
||||
OutputContract,
|
||||
ProducerReduction,
|
||||
ReadoutFusion,
|
||||
StageContract,
|
||||
StageKind,
|
||||
StagePath,
|
||||
@@ -96,6 +97,7 @@ __all__ = [
|
||||
"ExitDecision",
|
||||
"FfnExit",
|
||||
"FfnInputFusion",
|
||||
"ReadoutFusion",
|
||||
"DeferredFinalize",
|
||||
"GatedResidualState",
|
||||
"IHCState",
|
||||
|
||||
@@ -159,10 +159,14 @@ def get_attn_tp_context():
|
||||
return ATTN_TP_CONTEXT
|
||||
|
||||
|
||||
def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
|
||||
def attn_tp_gather(tensor: torch.Tensor, *, owned: bool = False) -> torch.Tensor:
|
||||
"""Gather this rank's attention-TP slice of the rows into all of them: in
|
||||
the local DP buffer, or in a buffer of its own for an output that
|
||||
outlives the next use of that buffer (``owned``)."""
|
||||
parallel = get_parallel()
|
||||
if batches_are_unpadded():
|
||||
# MLP sync, which sizes the local DP buffer for a batch, does not run.
|
||||
if owned or batches_are_unpadded():
|
||||
# MLP sync, which sizes the local DP buffer for a batch, does not run
|
||||
# for an unpadded one.
|
||||
gathered = tensor.new_empty(
|
||||
(tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1])
|
||||
)
|
||||
|
||||
@@ -187,6 +187,7 @@ def bind_entry(
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
attn_input_adapter: Optional[Callable] = None,
|
||||
attn_tp_gather: Optional[Callable] = None,
|
||||
) -> EntryPath:
|
||||
"""Bind the consumer half of an edge at construction time.
|
||||
|
||||
@@ -197,6 +198,8 @@ def bind_entry(
|
||||
cp_moves: Context-parallel strategy operations for this edge, if needed.
|
||||
enters_stack: Whether the read must initialize the stack's residual.
|
||||
attn_input_adapter: Hands an attention its input once it is on its rows.
|
||||
attn_tp_gather: The stage's implementation of a gather over attention
|
||||
TP, tried before the default one.
|
||||
|
||||
Returns:
|
||||
An EntryPath whose prepare accepts the actual update from the stream.
|
||||
@@ -218,6 +221,11 @@ def bind_entry(
|
||||
if edge.produced.update is None:
|
||||
raise ValueError("an arrival must declare its update capabilities")
|
||||
capabilities = (edge.produced.update.is_plain_add,)
|
||||
gather_input = (
|
||||
attn_tp_gather_input
|
||||
if attn_tp_gather is None
|
||||
else partial(attn_tp_gather_input, gather=attn_tp_gather)
|
||||
)
|
||||
paths = {
|
||||
capability: _bind_entry_path(
|
||||
edge,
|
||||
@@ -227,6 +235,7 @@ def bind_entry(
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
gather_input=gather_input,
|
||||
)
|
||||
for capability in capabilities
|
||||
}
|
||||
@@ -254,6 +263,7 @@ def _bind_entry_path(
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
attn_input_adapter: Optional[Callable] = None,
|
||||
gather_input: Callable = attn_tp_gather_input,
|
||||
) -> EntryPath:
|
||||
"""The consumer's half of ``edge``, chosen from the edge's declarations and
|
||||
the producer's update: what a value carries for a batch (a sum or a handoff
|
||||
@@ -277,6 +287,8 @@ def _bind_entry_path(
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
gather_input=gather_input,
|
||||
arrives_written=edge.arrives_written,
|
||||
)
|
||||
completed_step = None
|
||||
if edge.produced.always_partial:
|
||||
@@ -343,7 +355,12 @@ def _bind_entry_path(
|
||||
)
|
||||
|
||||
|
||||
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove:
|
||||
def bind_exit(
|
||||
edge: EdgeContract,
|
||||
*,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
attn_tp_gather: Optional[Callable] = None,
|
||||
) -> ExitMove:
|
||||
"""Bind the producer half of a layer or branch exit.
|
||||
|
||||
Args:
|
||||
@@ -351,6 +368,8 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit
|
||||
must outlive the producer; pipeline handoffs require a plain add
|
||||
or a residual already written by the producer.
|
||||
cp_moves: Context-parallel return operations, when the edge needs them.
|
||||
attn_tp_gather: The stage's implementation of a gather over attention
|
||||
TP, tried before the default one.
|
||||
|
||||
Returns:
|
||||
An ExitMove with fixed transport or a marker for batch-dependent DP
|
||||
@@ -373,6 +392,20 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Exit
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
def _read_fusions(read, owes, is_plain_add, *, scatters):
|
||||
"""The kernels a read supplies that complete ``owes`` on the rows this
|
||||
entry completes it on. They take the residual add themselves, so only a
|
||||
plain one."""
|
||||
if not is_plain_add:
|
||||
return ()
|
||||
return tuple(
|
||||
f
|
||||
for f in getattr(read, "completing_fusions", ())
|
||||
if f.completes is owes and f.scatters == scatters
|
||||
)
|
||||
|
||||
|
||||
@@ -387,6 +420,8 @@ def _select_entry_step(
|
||||
residual_joins_sum: bool,
|
||||
cp_moves: Optional[CpMoves],
|
||||
enters_stack: bool,
|
||||
gather_input: Callable = attn_tp_gather_input,
|
||||
arrives_written: bool = False,
|
||||
) -> Tuple[Callable, Optional[Callable]]:
|
||||
"""The steps into a stage for a value that is complete or owes the sum its
|
||||
producer always leaves, the fused kernels they try first, and the move
|
||||
@@ -447,14 +482,15 @@ def _select_entry_step(
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
if owes is SumGroup.ATTN_TP:
|
||||
step = partial(
|
||||
_attn_tp_reduce_scatter_update_read,
|
||||
read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True),
|
||||
)
|
||||
else:
|
||||
step = _attn_tp_slice_update_read
|
||||
return (
|
||||
partial(
|
||||
_attn_tp_reduce_scatter_update_read
|
||||
if owes is SumGroup.ATTN_TP
|
||||
else _attn_tp_slice_update_read,
|
||||
scatters_residual=residual != residual_to,
|
||||
read=read,
|
||||
),
|
||||
partial(step, scatters_residual=residual != residual_to, read=read),
|
||||
None,
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}:
|
||||
@@ -507,7 +543,7 @@ def _select_entry_step(
|
||||
return (partial(cp_moves.gather, gather=on_chunk), None)
|
||||
if gathered == {TokenAxis.ATTN_TP}:
|
||||
# A complete input on each rank's slice, gathered over attention TP
|
||||
# once it is read.
|
||||
# once it is read, unless one of the read's own kernels gathers it.
|
||||
if (
|
||||
owes is not None
|
||||
or residual != residual_to
|
||||
@@ -520,8 +556,13 @@ def _select_entry_step(
|
||||
pre_move=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
read_gathers=(
|
||||
getattr(read, "gathering_reads", ())
|
||||
if is_plain_add or arrives_written
|
||||
else ()
|
||||
),
|
||||
),
|
||||
attn_tp_gather_input,
|
||||
gather_input,
|
||||
)
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
@@ -572,11 +613,13 @@ def _select_entry_step(
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
fused = tuple(f for f in fusions if f.completes is owes)
|
||||
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
|
||||
return (
|
||||
partial(
|
||||
_reduce_update_read,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
read_fusions=read_fused,
|
||||
group=owes,
|
||||
read=read,
|
||||
),
|
||||
@@ -625,6 +668,7 @@ def _select_exit_move(
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
attn_tp_gather: Optional[Callable] = None,
|
||||
) -> ExitMove:
|
||||
"""How the FFN output reaches the rows the layer hands on: by undoing the
|
||||
attention-DP gather (the FFN exit and finish run that step), or else the
|
||||
@@ -638,7 +682,13 @@ def _select_exit_move(
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP}:
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return ExitMove(partial(update_attn_tp_gather_output, update=update))
|
||||
return ExitMove(
|
||||
partial(
|
||||
update_attn_tp_gather_output,
|
||||
update=update,
|
||||
gather=attn_tp_gather,
|
||||
)
|
||||
)
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP} and to in (residual, produced.layout):
|
||||
|
||||
@@ -47,6 +47,10 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_tp_gather_input,
|
||||
update_attn_tp_gather_output,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_input_default,
|
||||
_attn_input_scattered,
|
||||
@@ -186,6 +190,7 @@ class StagePlan:
|
||||
finishes_directly=False,
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
attn_tp_gather=None,
|
||||
):
|
||||
self.norm = norm
|
||||
self.edges = dict(variants)
|
||||
@@ -229,11 +234,16 @@ class StagePlan:
|
||||
cp_moves=edges.cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
attn_input_adapter=edges.attn_input_adapter,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
out = (
|
||||
ExitMove()
|
||||
if finishes_directly
|
||||
else bind_exit(edges.outgoing, cp_moves=edges.cp_moves)
|
||||
else bind_exit(
|
||||
edges.outgoing,
|
||||
cp_moves=edges.cp_moves,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
)
|
||||
self.paths[variant] = StagePath(
|
||||
entry=entry,
|
||||
@@ -243,6 +253,15 @@ class StagePlan:
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
if attn_tp_gather is not None and not any(
|
||||
getattr(path.output_move, "func", None) is update_attn_tp_gather_output
|
||||
or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
|
||||
for path in self.paths.values()
|
||||
):
|
||||
raise ValueError(
|
||||
"an attention-TP gather is declared, but no batch variant "
|
||||
"gathers over attention TP at this stage's boundaries"
|
||||
)
|
||||
|
||||
@property
|
||||
def incoming_residual_rows(self):
|
||||
@@ -334,6 +353,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
is_branch=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
writes_at_handoff=declaration.writes_at_handoff,
|
||||
attn_tp_gather=declaration.attn_tp_gather,
|
||||
finishes_directly=declaration.kind is StageKind.ATTENTION
|
||||
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
|
||||
**options,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"""Immutable stage declarations and bound input/output paths."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from typing import Callable, FrozenSet, Optional, Tuple
|
||||
from typing import Callable, FrozenSet, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -45,6 +45,10 @@ class ExitRows(Enum):
|
||||
|
||||
ATTENTION = auto()
|
||||
TBO_SPLIT = auto()
|
||||
# The rows the FFN ran on, also at the stack's end: an FFN on this rank's
|
||||
# attention-TP slice leaves its output there, and the model's final read
|
||||
# must read that slice and gather it.
|
||||
SLICE = auto()
|
||||
|
||||
|
||||
class BatchVariant(Enum):
|
||||
@@ -139,6 +143,8 @@ class EdgeContract(msgspec.Struct, frozen=True):
|
||||
arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for
|
||||
arriving contributions. Empty means use produced.update's capability.
|
||||
The actual update object travels with the residual stream.
|
||||
arrives_written: Whether the producer applies its update at its exit,
|
||||
so the stream arrives written with no residual add pending.
|
||||
"""
|
||||
|
||||
produced: OutputContract
|
||||
@@ -150,6 +156,7 @@ class EdgeContract(msgspec.Struct, frozen=True):
|
||||
residual_joins_sum: bool = False
|
||||
# Capabilities allowed to arrive from another layer, not its update object.
|
||||
arriving_plain_add: Tuple[bool, ...] = ()
|
||||
arrives_written: bool = False
|
||||
|
||||
|
||||
class FfnInputFusion(msgspec.Struct, frozen=True):
|
||||
@@ -168,6 +175,28 @@ class FfnInputFusion(msgspec.Struct, frozen=True):
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class ReadoutFusion(msgspec.Struct, frozen=True):
|
||||
"""A kernel a stage's read supplies (its ``completing_fusions``) that
|
||||
completes the sum its input owes together with the residual add, for a
|
||||
read that is not the residual's plain norm and so takes no FfnInputFusion.
|
||||
|
||||
``run(hidden_states, residual, forward_batch)`` returns the written stream,
|
||||
the completed sum plus the residual, which the read then reads; or None
|
||||
when it does not take the batch, before touching its inputs or starting a
|
||||
collective."""
|
||||
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]]
|
||||
# Whether it completes the sum onto this rank's attention-TP slice of the
|
||||
# rows (a reduce-scatter, given the whole residual or its slice) rather
|
||||
# than on every row.
|
||||
scatters: bool = False
|
||||
# Whether it also does the read: run(hidden_states, residual,
|
||||
# forward_batch, norm) then returns the read's (input, residual).
|
||||
reads: bool = False
|
||||
|
||||
|
||||
class CpMoves(msgspec.Struct, frozen=True):
|
||||
"""How a CP extend's rows reach an FFN that needs all of them and come back,
|
||||
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
|
||||
|
||||
@@ -7,7 +7,7 @@ import os
|
||||
import sys
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Mapping, Optional
|
||||
from typing import Callable, Mapping, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -260,6 +260,11 @@ class StageDeclaration:
|
||||
output_complete: Whether the FFN's compute completes its own output
|
||||
sum, so the exit owes none. For an FFN whose output sum is fused
|
||||
with a reduction its computation needs anyway.
|
||||
attn_tp_gather: Optional implementation of the gather over attention
|
||||
TP this stage's boundaries run: given this rank's contiguous slice
|
||||
of the rows, it returns them all, in rank order, or None to leave
|
||||
the gather to the boundary. Called on every batch, so it must be
|
||||
CUDA-graph safe.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
@@ -281,6 +286,7 @@ class StageDeclaration:
|
||||
exit_rows: Optional[ExitRows] = None
|
||||
writes_at_handoff: bool = False
|
||||
output_complete: bool = False
|
||||
attn_tp_gather: Optional[Callable] = None
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -325,6 +331,7 @@ def declare_attn(
|
||||
gathers_attn_tp_input=True,
|
||||
output_transform=None,
|
||||
tp_group=SumGroup.ATTN_TP,
|
||||
attn_tp_gather=None,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
@@ -339,6 +346,8 @@ def declare_attn(
|
||||
stage's input runs it, so no fused add + norm takes that input.
|
||||
Requires ALWAYS_PARTIAL.
|
||||
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
|
||||
attn_tp_gather: Implementation of this stage's gathers over attention
|
||||
TP, tried before the boundary's own (see StageDeclaration).
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -356,6 +365,7 @@ def declare_attn(
|
||||
reduction=reduction,
|
||||
gathers_attn_tp_input=gathers_attn_tp_input,
|
||||
tp_group=tp_group,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
@@ -369,6 +379,7 @@ def declare_ffn(
|
||||
dense_tp_size=None,
|
||||
exit_rows=None,
|
||||
output_complete=False,
|
||||
attn_tp_gather=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
@@ -385,6 +396,8 @@ def declare_ffn(
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
fused with a reduction it needs anyway; the exit then owes none.
|
||||
attn_tp_gather: Implementation of this stage's gathers over attention
|
||||
TP, tried before the boundary's own (see StageDeclaration).
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -398,6 +411,7 @@ def declare_ffn(
|
||||
dense_tp_size=dense_tp_size,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
output_complete=output_complete,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
@@ -423,8 +437,13 @@ def _resolve_stage(stage, variant, following=None):
|
||||
dense_tp_size=stage.dense_tp_size,
|
||||
output_complete=stage.output_complete,
|
||||
)
|
||||
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION:
|
||||
returned = attention
|
||||
elif exit_rows is ExitRows.SLICE:
|
||||
# The residual's rows during the FFN: this rank's slice for an FFN
|
||||
# on its own rows, else the attention's.
|
||||
returned = residual
|
||||
return declaration, residual, returned
|
||||
sp = variant is BatchVariant.SEQUENCE_PARALLEL
|
||||
scattered = variant is BatchVariant.INPUT_SCATTERED
|
||||
@@ -489,6 +508,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
exits, entries = {}, {}
|
||||
for variant in _active_variants():
|
||||
_, attention, local, _ = _row_layouts(variant)
|
||||
written = False
|
||||
if before is None:
|
||||
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
|
||||
owes = variant is BatchVariant.INPUT_SCATTERED
|
||||
@@ -548,6 +568,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
update=None,
|
||||
)
|
||||
residual, capabilities = returned, (before.update.is_plain_add,)
|
||||
written = before.update.applied_at_exit
|
||||
if after is None:
|
||||
continue
|
||||
decl, during, _ = _resolve_stage(after, variant)
|
||||
@@ -579,6 +600,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
during,
|
||||
residual_joins_sum=joins,
|
||||
arriving_plain_add=capabilities,
|
||||
arrives_written=written,
|
||||
)
|
||||
entries[variant] = edge
|
||||
if (
|
||||
|
||||
@@ -91,10 +91,27 @@ def tp_slice(hidden_states, residual):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class GatheredInput:
|
||||
"""A stage input its read already gathered over attention TP, which the
|
||||
entry's gather then passes on as it is."""
|
||||
|
||||
__slots__ = ("value",)
|
||||
|
||||
def __init__(self, value: torch.Tensor):
|
||||
self.value = value
|
||||
|
||||
|
||||
def attn_tp_gather_input(
|
||||
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
||||
forward_batch: ForwardBatch,
|
||||
gather: Optional[Callable] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Gather a stage input from this rank's attention-TP slice of the rows,
|
||||
trying the stage's ``gather`` first for a single tensor (see
|
||||
attn_tp_gather_with); an input its read already gathered (GatheredInput)
|
||||
passes through."""
|
||||
if isinstance(hidden_states, GatheredInput):
|
||||
return hidden_states.value
|
||||
parallel = get_parallel()
|
||||
if isinstance(hidden_states, tuple):
|
||||
gathered_hidden_states = []
|
||||
@@ -118,7 +135,7 @@ def attn_tp_gather_input(
|
||||
gathered_hidden_states.append(output)
|
||||
return tuple(gathered_hidden_states)
|
||||
|
||||
return attn_tp_gather(hidden_states)
|
||||
return attn_tp_gather_with(hidden_states, gather)
|
||||
|
||||
|
||||
def attn_tp_reduce_scatter(
|
||||
@@ -328,10 +345,22 @@ def update_attn_tp_gather_output(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
gather: Optional[Callable] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Write the output into the residual on this rank's slice, then gather it
|
||||
over attention TP (see attn_tp_gather_with)."""
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
return attn_tp_gather(hidden_states), None
|
||||
return attn_tp_gather_with(hidden_states, gather), None
|
||||
|
||||
|
||||
def attn_tp_gather_with(
|
||||
hidden_states: torch.Tensor, gather: Optional[Callable]
|
||||
) -> torch.Tensor:
|
||||
"""Gather this rank's attention-TP slice of the rows into all of them: in
|
||||
the stage's own ``gather`` when it takes the batch, else the boundary's."""
|
||||
gathered = gather(hidden_states) if gather is not None else None
|
||||
return attn_tp_gather(hidden_states) if gathered is None else gathered
|
||||
|
||||
|
||||
def residual_slice_output(
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
get_attn_tp_context,
|
||||
tp_gather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
@@ -66,6 +67,7 @@ if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
||||
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
GatheredInput,
|
||||
attn_tp_all_reduce,
|
||||
attn_tp_reduce_scatter,
|
||||
dp_gather,
|
||||
@@ -74,6 +76,25 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
)
|
||||
|
||||
|
||||
def _run_read_fusions(
|
||||
read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call
|
||||
):
|
||||
"""The first of the read's own kernels (ReadoutFusion) that takes the
|
||||
batch: one that completes the sum with the residual add hands the written
|
||||
stream to ``read``; one that also reads returns the read. None when every
|
||||
kernel declines."""
|
||||
for fused in read_fusions:
|
||||
if fused.reads:
|
||||
result = fused.run(hidden_states, residual, forward_batch, norm)
|
||||
if result is not None:
|
||||
return result
|
||||
continue
|
||||
stream = fused.run(hidden_states, residual, forward_batch)
|
||||
if stream is not None:
|
||||
return read.read(stream, norm, **call)
|
||||
return None
|
||||
|
||||
|
||||
def _reduce_update_read(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -83,6 +104,7 @@ def _reduce_update_read(
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
read_fusions: Tuple[ReadoutFusion, ...] = (),
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
@@ -90,11 +112,24 @@ def _reduce_update_read(
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Complete the sum the input owes over ``group`` on the rows it is on,
|
||||
unless one of ``fusions`` does it with the residual add and the norm, then
|
||||
write it into the residual and read the input (an attention's read takes
|
||||
unless one of the read's ``read_fusions`` does it with the residual add,
|
||||
or one of ``fusions`` with the residual add and the norm, then write it
|
||||
into the residual and read the input (an attention's read takes
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if gathers_residual:
|
||||
residual = update.gather_residual_attn_tp(residual)
|
||||
result = _run_read_fusions(
|
||||
read_fusions,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
@@ -315,6 +350,7 @@ def _update_read(
|
||||
pre_move: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: ResidualReadout,
|
||||
read_gathers: Tuple[Callable, ...] = (),
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
@@ -322,12 +358,19 @@ def _update_read(
|
||||
"""Complete what the input owes by construction (``pre_move``), then
|
||||
write the previous stage's output into the residual and read the stage's
|
||||
input with ``norm``. The layer stack's first stage (``enters_stack``)
|
||||
starts its residual from its input."""
|
||||
starts its residual from its input. The first of ``read_gathers`` that
|
||||
takes the batch reads and gathers the input itself; its result is wrapped
|
||||
in GatheredInput so the entry's gather passes it through."""
|
||||
enters = residual is None and enters_stack
|
||||
if pre_move is not None:
|
||||
hidden_states, residual = pre_move(hidden_states, residual)
|
||||
if enters:
|
||||
hidden_states, residual = read.init_residual(hidden_states), None
|
||||
for fused in read_gathers:
|
||||
result = fused(hidden_states, residual, norm)
|
||||
if result is not None:
|
||||
gathered, residual = result
|
||||
return GatheredInput(gathered), residual
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
@@ -349,9 +392,19 @@ def _attn_tp_reduce_scatter_update_read(
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read_fusions: Tuple[ReadoutFusion, ...] = (),
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
):
|
||||
"""Complete the attention-TP sum onto this rank's slice, slice the residual
|
||||
the same way, then write the output into it and read the input; the first
|
||||
of the read's ``read_fusions`` that takes the batch does all of this
|
||||
instead."""
|
||||
result = _run_read_fusions(
|
||||
read_fusions, hidden_states, residual, forward_batch, norm, read=read
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
hidden_states = attn_tp_reduce_scatter(hidden_states)
|
||||
if scatters_residual and residual is not None:
|
||||
residual = update.slice_residual_attn_tp(residual)
|
||||
|
||||
@@ -70,6 +70,13 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: Read is normalization/optional quantization without changing
|
||||
the residual, allowing compatible fused add+norm implementations.
|
||||
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
|
||||
completing_fusions: Optional. ReadoutFusion kernels that complete the
|
||||
sum the input owes with the residual add ahead of this read, tried
|
||||
before the boundary's own all-reduce or reduce-scatter.
|
||||
gathering_reads: Optional. Callables ``(hidden_states, residual,
|
||||
norm)`` that read the input on this rank's attention-TP slice and
|
||||
gather it over attention TP, returning ``(input, residual)``, or
|
||||
None when they do not take the batch.
|
||||
|
||||
init_residual initializes the stack residual. read consumes an
|
||||
already-written residual; update_and_read first applies the actual
|
||||
|
||||
@@ -14,11 +14,14 @@
|
||||
"""A residual read that aggregates a bank of snapshots of itself."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attn_residual import AttnResidual
|
||||
from sglang.srt.layers.layer_boundary import ops
|
||||
from sglang.srt.layers.layer_boundary.contracts import ReadoutFusion
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -87,6 +90,12 @@ class AttnBankState:
|
||||
ffn_score_proj: torch.nn.Module
|
||||
ffn_score_norm: torch.nn.Module
|
||||
writes_block: bool = False
|
||||
# Kernels that complete the attention output's sum with the pending add
|
||||
# ahead of the FFN read.
|
||||
ffn_input_fusions: Tuple[ReadoutFusion, ...] = ()
|
||||
# A read on this rank's attention-TP slice first tries the bank's kernels
|
||||
# that also run the collective around it.
|
||||
fuses_slice_collectives: bool = False
|
||||
|
||||
def _aggregate(
|
||||
self, contribution, residual, norm, *, score_proj, score_norm, write
|
||||
@@ -131,6 +140,42 @@ class AttnBankState:
|
||||
write=False,
|
||||
)
|
||||
|
||||
def reduce_scatter_and_read_ffn_input(
|
||||
self, hidden_states, residual, forward_batch, norm
|
||||
):
|
||||
parallel = get_parallel()
|
||||
rows, rest = divmod(hidden_states.shape[0], parallel.attn_tp_size)
|
||||
if rows == 0 or rest:
|
||||
return None
|
||||
start = parallel.attn_tp_rank * rows
|
||||
return self.bank.require().forward_sp_reduce_scatter(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.ffn_score_proj,
|
||||
self.ffn_score_norm,
|
||||
norm,
|
||||
rows=slice(start, start + rows),
|
||||
)
|
||||
|
||||
def read_and_gather_attn_input(self, contribution, residual, norm):
|
||||
bank = self.bank.require()
|
||||
rows = _bank_rows(bank, contribution)
|
||||
if rows is None:
|
||||
return None
|
||||
read = bank.forward_sp_all_gather(
|
||||
contribution,
|
||||
residual,
|
||||
self.attn_score_proj,
|
||||
self.attn_score_norm,
|
||||
norm,
|
||||
rows=rows,
|
||||
write=self.writes_block,
|
||||
)
|
||||
if read is None:
|
||||
return None
|
||||
normed, residual = read
|
||||
return normed, None if self.writes_block else residual
|
||||
|
||||
def residual_ops(self) -> LayerResidualOps:
|
||||
return LayerResidualOps(
|
||||
attn_readout=_AttnReadout(self),
|
||||
@@ -171,6 +216,13 @@ class _AttnReadout(_BankReadout):
|
||||
"""The attention input: the bank aggregation and this layer's input norm.
|
||||
A write layer snapshots the residual this read forms."""
|
||||
|
||||
@property
|
||||
def gathering_reads(self):
|
||||
state = self.state
|
||||
return (
|
||||
(state.read_and_gather_attn_input,) if state.fuses_slice_collectives else ()
|
||||
)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
self._reject(None, quant_format)
|
||||
if post_residual_addition is not None:
|
||||
@@ -188,6 +240,19 @@ class _FfnReadout(_BankReadout):
|
||||
"""The FFN input: the attention output's add folded into the bank
|
||||
aggregation, and this layer's post-attention norm."""
|
||||
|
||||
@property
|
||||
def completing_fusions(self) -> Tuple[ReadoutFusion, ...]:
|
||||
state = self.state
|
||||
if not state.fuses_slice_collectives:
|
||||
return state.ffn_input_fusions
|
||||
reads = ReadoutFusion(
|
||||
SumGroup.ATTN_TP,
|
||||
state.reduce_scatter_and_read_ffn_input,
|
||||
scatters=True,
|
||||
reads=True,
|
||||
)
|
||||
return (reads, *state.ffn_input_fusions)
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
# The attention read of a write layer leaves the residual empty, so the
|
||||
# attention output alone is the head.
|
||||
@@ -202,22 +267,40 @@ class _FfnReadout(_BankReadout):
|
||||
class AttnBankOutputRead:
|
||||
"""The layer stack's terminal read: the bank aggregation with the output
|
||||
side's scoring parameters, then the final norm. A callable final norm for
|
||||
`residual_batch.final_norm`."""
|
||||
`residual_batch.final_norm`.
|
||||
|
||||
def __init__(self, bank: AttnBank, score_proj, score_norm, norm):
|
||||
A stack that ends on this rank's attention-TP slice of the rows is read
|
||||
there, against the bank rows this rank wrote, and what it read is
|
||||
gathered: by the bank's kernel that does both when it takes the batch,
|
||||
else by ``attn_tp_gather`` when that does, else over the attention-TP
|
||||
group."""
|
||||
|
||||
def __init__(
|
||||
self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None
|
||||
):
|
||||
self.bank = bank
|
||||
self.score_proj = score_proj
|
||||
self.score_norm = score_norm
|
||||
self.norm = norm
|
||||
self.attn_tp_gather = attn_tp_gather
|
||||
|
||||
def __call__(self, hidden_states, residual=None):
|
||||
bank = self.bank.require()
|
||||
normed, head = bank.forward(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.score_proj,
|
||||
self.score_norm,
|
||||
self.norm,
|
||||
rows=_bank_rows(bank, hidden_states),
|
||||
rows = _bank_rows(bank, hidden_states)
|
||||
read_args = (hidden_states, residual, self.score_proj, self.score_norm)
|
||||
read = (
|
||||
None
|
||||
if rows is None
|
||||
else bank.forward_sp_all_gather(*read_args, self.norm, rows=rows)
|
||||
)
|
||||
if read is None:
|
||||
read = bank.forward(*read_args, self.norm, rows=rows)
|
||||
if rows is not None:
|
||||
read = (self._gather(read[0]), read[1])
|
||||
normed, head = read
|
||||
return normed if residual is None else (normed, head)
|
||||
|
||||
def _gather(self, normed):
|
||||
gathered = None if self.attn_tp_gather is None else self.attn_tp_gather(normed)
|
||||
# The output outlives the layer stack.
|
||||
return ops.attn_tp_gather(normed, owned=True) if gathered is None else gathered
|
||||
|
||||
@@ -51,6 +51,8 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
ReadoutFusion,
|
||||
SumGroup,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -142,7 +144,9 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils import is_hip, is_npu, make_pp_layers
|
||||
from sglang.srt.utils.common import (
|
||||
BumpAllocator,
|
||||
@@ -428,6 +432,48 @@ def _o_proj_takes_output(o_proj: RowParallelLinear) -> bool:
|
||||
return getattr(o_proj.quant_method, "apply_into", None) is not None
|
||||
|
||||
|
||||
def _k3_all_reduce_add(hidden_states, residual, forward_batch):
|
||||
"""o_proj's attention-TP sum with the pending residual add in K3's fused
|
||||
all-reduce, for an output o_proj wrote into its multicast buffer."""
|
||||
if k3_ar_fusion.find_mc_ptr(hidden_states) is None:
|
||||
return None
|
||||
return k3_ar_fusion.all_reduce(hidden_states, residual)
|
||||
|
||||
|
||||
def _k3_reduce_scatter_add(hidden_states, residual, forward_batch):
|
||||
"""o_proj's attention-TP sum onto this rank's slice with the pending
|
||||
residual add, in K3's tuned reduce-scatter when it takes the batch."""
|
||||
return k3_sp_collective.reduce_scatter_res(hidden_states, residual)
|
||||
|
||||
|
||||
def _route_sp_o_proj_output(o_proj: RowParallelLinear) -> None:
|
||||
"""o_proj under SP-MoE."""
|
||||
# o_proj emits TP-partial sums, which the stage boundary completes.
|
||||
o_proj.reduce_results = False
|
||||
if k3_sp_collective.enabled():
|
||||
# The table selects NVLS pull RS for larger token buckets.
|
||||
# Only those o_proj outputs come from the persistent symmetric
|
||||
# buffer; small push RS keeps the regular graph allocator.
|
||||
_sp_inner_o_proj_forward = o_proj.forward
|
||||
|
||||
def _sp_o_proj_forward(x, *args, **kwargs):
|
||||
output_rows = x.shape[0]
|
||||
if k3_sp_collective.requires_symmetric_rs(
|
||||
output_rows, x.device, x.element_size()
|
||||
):
|
||||
output = k3_sp_collective.get_o_proj_output_buffer(
|
||||
output_rows, x.dtype, o_proj.output_size
|
||||
)
|
||||
result = _sp_inner_o_proj_forward(
|
||||
x, *args, output_tensor=output[: x.shape[0]], **kwargs
|
||||
)
|
||||
k3_sp_collective.register_o_proj_output(result[0], output)
|
||||
return result
|
||||
return _sp_inner_o_proj_forward(x, *args, **kwargs)
|
||||
|
||||
o_proj.forward = _sp_o_proj_forward
|
||||
|
||||
|
||||
def _k3_symm_o_proj_out(o_proj: RowParallelLinear, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Symmetric storage for o_proj's TP-partial output; the fused attention
|
||||
all-reduce reduces it in place."""
|
||||
@@ -1546,6 +1592,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
all_reduce_fusion: bool = False,
|
||||
bfa_alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
reduce_results: bool = True,
|
||||
sp_moe: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -1818,7 +1865,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
# the fused AR reduces o_proj's output in place out of a symmetric
|
||||
# buffer, which needs the GEMM to write into caller-owned storage
|
||||
self.all_reduce_fusion = False
|
||||
self.o_proj.reduce_results = True
|
||||
self.o_proj.reduce_results = reduce_results
|
||||
self.o_proj.use_dp_attention_reduce = True
|
||||
conv_weights = self.qkv_conv1d.weight.squeeze(1)
|
||||
bias = self.qkv_conv1d.bias
|
||||
@@ -1841,6 +1888,8 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
# Set by _prepare_fused_decode() once weights are loaded.
|
||||
self._kda_fused_decode_ready = False
|
||||
self._kda_hip_fused_decode_ready = False
|
||||
if sp_moe:
|
||||
_route_sp_o_proj_output(self.o_proj)
|
||||
|
||||
def forward_qkvbfg(self, hidden_states: torch.Tensor):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
@@ -2200,6 +2249,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
gate_alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
reduce_results: bool = True,
|
||||
sp_moe: bool = False,
|
||||
) -> None:
|
||||
# ModelSlim can quantize K3 latent projections while still storing
|
||||
# MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V.
|
||||
@@ -2276,7 +2326,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
# the fused AR reduces o_proj's output in place out of a symmetric
|
||||
# buffer, which needs the GEMM to write into caller-owned storage
|
||||
self.all_reduce_fusion = False
|
||||
self.o_proj.reduce_results = True
|
||||
self.o_proj.reduce_results = reduce_results
|
||||
self.o_proj.use_dp_attention_reduce = True
|
||||
if self.all_reduce_fusion:
|
||||
# Hand the GEMM a slice of the persistent symmetric buffer
|
||||
@@ -2350,6 +2400,8 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
return _orig_o_proj_forward(x, *args, **kwargs)
|
||||
|
||||
self.o_proj.forward = _gated_o_proj_forward
|
||||
if sp_moe:
|
||||
_route_sp_o_proj_output(self.o_proj)
|
||||
|
||||
@staticmethod
|
||||
def _split_kv_b_weight_loader(param, loaded_weight) -> None:
|
||||
@@ -2445,17 +2497,34 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the layers build stage boundaries, which is the same for every
|
||||
layer of a stack. The layer's own communication still runs for: an
|
||||
attention-residual bank whose o_proj all-reduce is fused with the pending
|
||||
add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned
|
||||
SP collectives, which the sharded carry also needs."""
|
||||
if _fuses_attn_all_reduce(config):
|
||||
def _carries_bank_slices(config: KimiLinearConfig) -> bool:
|
||||
"""Whether consecutive SP-MoE layers keep the stream and the
|
||||
attention-residual bank on each rank's attention-TP shard of the rows
|
||||
(SGLANG_K3_SP_ATTN_RES): each layer reads its attention input there, then
|
||||
gathers what it read. A rank then holds only its own rows of the bank, so
|
||||
neither a later pipeline rank nor a draft model's capture of the target's
|
||||
hidden states can take them; it is decided at construction. Nor can a
|
||||
dense layer after an SP-MoE one, whose FFN reads every row of the bank:
|
||||
every layer from the first MoE layer on must be one."""
|
||||
if not (
|
||||
_shards_moe_rows()
|
||||
and config.attn_res_block_size is not None
|
||||
and k3_sp_collective.enabled()
|
||||
and envs.SGLANG_K3_SP_ATTN_RES.get()
|
||||
and get_parallel().pp_size == 1
|
||||
):
|
||||
return False
|
||||
if not _shards_moe_rows():
|
||||
return True
|
||||
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
|
||||
moe = [_is_moe_layer(config, idx) for idx in range(config.num_hidden_layers)]
|
||||
if True not in moe or not all(moe[moe.index(True) :]):
|
||||
return False
|
||||
spec = SpeculativeAlgorithm.from_string(get_spec().speculative_algorithm)
|
||||
return not (spec.is_eagle3() or spec.is_dflash_family())
|
||||
|
||||
|
||||
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the layers build stage boundaries, which every configuration
|
||||
does."""
|
||||
return True
|
||||
|
||||
|
||||
class KimiK3DecoderLayer(nn.Module):
|
||||
@@ -2500,6 +2569,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.self_attn",
|
||||
all_reduce_fusion=self.all_reduce_fusion,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
sp_moe=self._sp_moe,
|
||||
# Shared with the MLA gate stream: KDA and MLA layers never
|
||||
# run concurrently within one forward, so the stream is free.
|
||||
bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None),
|
||||
@@ -2512,6 +2582,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.self_attn",
|
||||
all_reduce_fusion=self.all_reduce_fusion,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
sp_moe=self._sp_moe,
|
||||
alt_stream=alt_streams[1] if alt_streams is not None else None,
|
||||
gate_alt_stream=alt_streams[2] if alt_streams is not None else None,
|
||||
)
|
||||
@@ -2571,35 +2642,6 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.mlp_res_proj",
|
||||
)
|
||||
|
||||
if self._sp_moe:
|
||||
# o_proj emits TP-partial sums; _finish_attn_reduce completes the
|
||||
# reduction (RS on the clean attn-res path, AR on fallbacks).
|
||||
o_proj = getattr(self.self_attn, "o_proj", None)
|
||||
assert o_proj is not None, "SP-MoE requires attention exposing o_proj"
|
||||
o_proj.reduce_results = False
|
||||
if k3_sp_collective.enabled():
|
||||
# The table selects NVLS pull RS for larger token buckets.
|
||||
# Only those o_proj outputs come from the persistent symmetric
|
||||
# buffer; small push RS keeps the regular graph allocator.
|
||||
_sp_inner_o_proj_forward = o_proj.forward
|
||||
|
||||
def _sp_o_proj_forward(x, *args, **kwargs):
|
||||
output_rows = x.shape[0]
|
||||
if k3_sp_collective.requires_symmetric_rs(
|
||||
output_rows, x.device, x.element_size()
|
||||
):
|
||||
output = k3_sp_collective.get_o_proj_output_buffer(
|
||||
output_rows, x.dtype, o_proj.output_size
|
||||
)
|
||||
result = _sp_inner_o_proj_forward(
|
||||
x, *args, output_tensor=output[: x.shape[0]], **kwargs
|
||||
)
|
||||
k3_sp_collective.register_o_proj_output(result[0], output)
|
||||
return result
|
||||
return _sp_inner_o_proj_forward(x, *args, **kwargs)
|
||||
|
||||
o_proj.forward = _sp_o_proj_forward
|
||||
|
||||
# A latent MoE on this rank's rows adds the residual in its tail add and
|
||||
# writes the next stream itself; one gathered over attention DP returns
|
||||
# its output to this rank's tokens first.
|
||||
@@ -2611,56 +2653,98 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
and not (self._dp_attention and not self.mlp._ep_a2a)
|
||||
)
|
||||
if self._stage_boundaries:
|
||||
# Under attention DP the FFN input is read on this rank's rows
|
||||
# once the attention's sum is complete, then gathered.
|
||||
attn_ops = {}
|
||||
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
|
||||
if self.use_attn_residuals:
|
||||
bank_ops = AttnBankState(
|
||||
attn_bank,
|
||||
self.self_attention_res_proj,
|
||||
self.self_attention_res_norm,
|
||||
self.mlp_res_proj,
|
||||
self.mlp_res_norm,
|
||||
writes_block=self.is_block_write_layer,
|
||||
).residual_ops()
|
||||
attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update)
|
||||
ffn_ops = dict(
|
||||
read=bank_ops.ffn_readout,
|
||||
update=(
|
||||
REPLACE_AT_EXIT
|
||||
if self._ffn_writes_stream
|
||||
else bank_ops.ffn_update
|
||||
),
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(**attn_ops), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
**ffn_ops,
|
||||
sparse=self._is_moe_layer,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
|
||||
# The TP width the dense MLP is built with.
|
||||
dense_tp_size=(
|
||||
None
|
||||
if self._is_moe_layer
|
||||
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
|
||||
),
|
||||
# SP-MoE runs on this rank's attention-TP shard of the
|
||||
# rows; on the bank path, whose reads write the bank on
|
||||
# every row, its output returns to all of them.
|
||||
exit_rows=(
|
||||
ExitRows.ATTENTION
|
||||
if self._sp_moe and self.use_attn_residuals
|
||||
else None
|
||||
),
|
||||
# A latent MoE completes its output sum together with
|
||||
# the latent reduction its norm needs.
|
||||
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
self._declare_stages(config, layer_idx, attn_bank)
|
||||
|
||||
def _declare_stages(self, config, layer_idx, attn_bank):
|
||||
"""Declare this layer's attention and FFN stages: the reads and updates
|
||||
around them, the rows the FFN takes and leaves, and the kernels K3
|
||||
supplies for the bank path."""
|
||||
# Under attention DP the FFN input is read on this rank's rows
|
||||
# once the attention's sum is complete, then gathered.
|
||||
attn_ops = {}
|
||||
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
|
||||
carries = _carries_bank_slices(config)
|
||||
bank_sp_moe = self._sp_moe and self.use_attn_residuals
|
||||
tuned_gather = (
|
||||
k3_sp_collective.all_gather if k3_sp_collective.enabled() else None
|
||||
)
|
||||
if self.use_attn_residuals:
|
||||
bank_ops = AttnBankState(
|
||||
attn_bank,
|
||||
self.self_attention_res_proj,
|
||||
self.self_attention_res_norm,
|
||||
self.mlp_res_proj,
|
||||
self.mlp_res_norm,
|
||||
writes_block=self.is_block_write_layer,
|
||||
ffn_input_fusions=self._ffn_input_fusions(),
|
||||
fuses_slice_collectives=self._sp_moe
|
||||
and k3_sp_collective.enabled()
|
||||
and envs.SGLANG_K3_SP_ATTN_RES.get(),
|
||||
).residual_ops()
|
||||
attn_ops = dict(
|
||||
read=bank_ops.attn_readout,
|
||||
update=bank_ops.attn_update,
|
||||
# An input read on the shard an SP-MoE layer left is gathered
|
||||
# in K3's tuned all-gather when it takes the batch.
|
||||
attn_tp_gather=(
|
||||
tuned_gather
|
||||
if carries and _is_moe_layer(config, layer_idx - 1)
|
||||
else None
|
||||
),
|
||||
)
|
||||
ffn_ops = dict(
|
||||
read=bank_ops.ffn_readout,
|
||||
update=(
|
||||
REPLACE_AT_EXIT if self._ffn_writes_stream else bank_ops.ffn_update
|
||||
),
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(**attn_ops), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
**ffn_ops,
|
||||
sparse=self._is_moe_layer,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
|
||||
# The TP width the dense MLP is built with.
|
||||
dense_tp_size=(
|
||||
None
|
||||
if self._is_moe_layer
|
||||
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
|
||||
),
|
||||
# SP-MoE runs on this rank's attention-TP shard of the
|
||||
# rows; on the bank path, whose reads write the bank on
|
||||
# every row, its output returns to all of them, unless
|
||||
# the bank stays on the shard, read there to the end.
|
||||
exit_rows=(
|
||||
(ExitRows.SLICE if carries else ExitRows.ATTENTION)
|
||||
if bank_sp_moe
|
||||
else None
|
||||
),
|
||||
# A latent MoE completes its output sum together with
|
||||
# the latent reduction its norm needs.
|
||||
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
|
||||
# The bank path's gather back to every row, in K3's
|
||||
# tuned all-gather when it takes the batch.
|
||||
attn_tp_gather=(
|
||||
tuned_gather if bank_sp_moe and not carries else None
|
||||
),
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
)
|
||||
|
||||
def _ffn_input_fusions(self):
|
||||
"""K3's kernels that complete the attention output's sum with the
|
||||
pending residual add ahead of the bank's FFN read: the fused
|
||||
all-reduce on every row, or the tuned reduce-scatter onto this rank's
|
||||
attention-TP shard (SP-MoE)."""
|
||||
if self.all_reduce_fusion:
|
||||
return (ReadoutFusion(SumGroup.ATTN_TP, _k3_all_reduce_add),)
|
||||
if self._sp_moe and k3_sp_collective.enabled():
|
||||
return (
|
||||
ReadoutFusion(SumGroup.ATTN_TP, _k3_reduce_scatter_add, scatters=True),
|
||||
)
|
||||
return ()
|
||||
|
||||
def _finish_attn_reduce(
|
||||
self,
|
||||
@@ -2949,6 +3033,7 @@ class KimiK3LinearModel(nn.Module):
|
||||
self.dspark_layers_to_capture: Optional[list[int]] = None
|
||||
self._dp_attention = is_dp_attention_enabled()
|
||||
self._stage_boundaries = _uses_stage_boundaries(config)
|
||||
self.carries_bank_slices = _carries_bank_slices(config)
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
embedding_quant_config = (
|
||||
@@ -3016,6 +3101,11 @@ class KimiK3LinearModel(nn.Module):
|
||||
self.output_attn_res_proj,
|
||||
self.output_attn_res_norm,
|
||||
self.norm,
|
||||
attn_tp_gather=(
|
||||
k3_sp_collective.all_gather
|
||||
if self.carries_bank_slices
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -3359,6 +3449,12 @@ class KimiK3LinearForCausalLM(nn.Module):
|
||||
raise ValueError(
|
||||
"DSPARK requires explicit layer_ids for aux hidden capture."
|
||||
)
|
||||
if self.model.carries_bank_slices:
|
||||
raise RuntimeError(
|
||||
"the model keeps each rank's shard of the rows across SP-MoE "
|
||||
"layers (SGLANG_K3_SP_ATTN_RES), which leaves no layer output "
|
||||
"to capture whole"
|
||||
)
|
||||
self.capture_aux_hidden_states = True
|
||||
self.model.dspark_layers_to_capture = list(layer_ids)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user