[Refactor] Remove Kimi K3's own layer communication and let the consumer set an FFN's exit (#42936)

This commit is contained in:
Cheng Wan
2026-10-09 01:27:46 -07:00
committed by GitHub
parent 305ad5503d
commit dc0b394471
16 changed files with 424 additions and 730 deletions
@@ -47,10 +47,6 @@ from sglang.srt.layers.layer_boundary.layout import (
batches_are_unpadded,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.ops import (
attn_tp_gather_input,
update_attn_tp_gather_output,
)
from sglang.srt.layers.layer_boundary.prepare import (
_attn_input_default,
_attn_input_scattered,
@@ -191,6 +187,7 @@ class StagePlan:
qkv_latent_func=None,
fusions=None,
attn_tp_gather=None,
exit_gather=None,
):
self.norm = norm
self.edges = dict(variants)
@@ -242,7 +239,7 @@ class StagePlan:
else bind_exit(
edges.outgoing,
cp_moves=edges.cp_moves,
attn_tp_gather=attn_tp_gather,
attn_tp_gather=exit_gather,
)
)
self.paths[variant] = StagePath(
@@ -253,15 +250,6 @@ class StagePlan:
returns_over_dp=out.returns_over_dp,
writes_at_handoff=writes_at_handoff,
)
if attn_tp_gather is not None and not any(
getattr(path.output_move, "func", None) is update_attn_tp_gather_output
or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
for path in self.paths.values()
):
raise ValueError(
"an attention-TP gather is declared, but no batch variant "
"gathers over attention TP at this stage's boundaries"
)
@property
def incoming_residual_rows(self):
@@ -319,7 +307,7 @@ class StagePlan:
)
def _bind_stage(declaration, norm, incoming, outgoing, **options):
def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options):
if incoming.consumer != declaration or outgoing.producer != declaration:
raise ValueError("connections do not match the stage declaration")
if incoming.entries.keys() != outgoing.exits.keys():
@@ -354,6 +342,13 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
terminal=declaration.terminal,
writes_at_handoff=declaration.writes_at_handoff,
attn_tp_gather=declaration.attn_tp_gather,
# The exit runs its consumer's gather: the next stage's, or the
# final read's.
exit_gather=getattr(
outgoing.consumer if outgoing.consumer is not None else final_read,
"attn_tp_gather",
None,
),
finishes_directly=declaration.kind is StageKind.ATTENTION
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
**options,
@@ -45,9 +45,9 @@ class ExitRows(Enum):
ATTENTION = auto()
TBO_SPLIT = auto()
# The rows the FFN ran on, also at the stack's end: an FFN on this rank's
# attention-TP slice leaves its output there, and the model's final read
# must read that slice and gather it.
# The rows the FFN ran on, also at the stack's end: the layer stack's last
# FFN, when the model's final read reads this rank's attention-TP slice
# and gathers it.
SLICE = auto()
@@ -44,6 +44,10 @@ from sglang.srt.layers.layer_boundary.layout import (
moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.layer_boundary.ops import (
attn_tp_gather_input,
update_attn_tp_gather_output,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import (
@@ -261,10 +265,14 @@ class StageDeclaration:
sum, so the exit owes none. For an FFN whose output sum is fused
with a reduction its computation needs anyway.
attn_tp_gather: Optional implementation of the gather over attention
TP this stage's boundaries run: given this rank's contiguous slice
of the rows, it returns them all, in rank order, or None to leave
the gather to the boundary. Called on every batch, so it must be
CUDA-graph safe.
TP that brings rows into this stage: at its own entry, or at its
producer's exit, which runs its consumer's gather. Given this rank's
contiguous slice of the rows, it returns them all, in rank order,
or None to leave the gather to the boundary. Called on every batch,
so it must be CUDA-graph safe. Across a pipeline boundary the
producer's rank takes it from the neighbouring layer it builds on
the meta device, so it may use only the communication state of the
rank it runs on, not the declaring layer's weights or buffers.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
prepared_from: Declaration whose already-read input a branch reuses.
@@ -346,8 +354,8 @@ def declare_attn(
stage's input runs it, so no fused add + norm takes that input.
Requires ALWAYS_PARTIAL.
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
attn_tp_gather: Implementation of this stage's gathers over attention
TP, tried before the boundary's own (see StageDeclaration).
attn_tp_gather: Implementation of the gather over attention TP into
this stage, tried before the boundary's own (see StageDeclaration).
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -379,7 +387,6 @@ def declare_ffn(
dense_tp_size=None,
exit_rows=None,
output_complete=False,
attn_tp_gather=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -396,8 +403,6 @@ def declare_ffn(
the adjacent FFN kinds and TBO configuration.
output_complete: Whether the compute completes its own output sum,
fused with a reduction it needs anyway; the exit then owes none.
attn_tp_gather: Implementation of this stage's gathers over attention
TP, tried before the boundary's own (see StageDeclaration).
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -411,7 +416,6 @@ def declare_ffn(
dense_tp_size=dense_tp_size,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
output_complete=output_complete,
attn_tp_gather=attn_tp_gather,
)
@@ -438,7 +442,10 @@ def _resolve_stage(stage, variant, following=None):
output_complete=stage.output_complete,
)
exit_rows = resolve_exit_rows(stage.exit_rows)
if exit_rows is ExitRows.ATTENTION:
if exit_rows is ExitRows.ATTENTION or (
following is not None
and getattr(following.read, "reads_after_attn_tp_gather", False)
):
returned = attention
elif exit_rows is ExitRows.SLICE:
# The residual's rows during the FFN: this rank's slice for an FFN
@@ -674,12 +681,13 @@ class _LayerStack:
"""A layer stack under construction: what was appended, in order, and how
to reach the layers other pipeline ranks hold on either side of it."""
__slots__ = ("appends", "previous_layers", "next_layers")
__slots__ = ("appends", "previous_layers", "next_layers", "final_read")
def __init__(self, previous_layers=(), next_layers=()):
def __init__(self, previous_layers=(), next_layers=(), final_read=None):
self.appends = []
self.previous_layers = previous_layers
self.next_layers = next_layers
self.final_read = final_read
# The stack being built; layer_stack saves and restores an outer one.
@@ -687,7 +695,7 @@ _stack: Optional[_LayerStack] = None
@contextlib.contextmanager
def layer_stack(*, previous_layers=(), next_layers=()):
def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
"""Open a layer stack that append_stages extends in order.
Every stage binds when the stack closes, once its producer and its
@@ -702,10 +710,15 @@ def layer_stack(*, previous_layers=(), next_layers=()):
stack's first. What they build is discarded.
next_layers: Likewise for the layers after this stack, whose first
declared stage is the consumer of this stack's last.
final_read: The model's final read of the stack's output, when it is
not a plain final norm: the consumer of the last stage when no
later layer declares one. Its ``attn_tp_gather`` gathers the rows
that stage leaves on this rank's attention-TP slice, and with
``reads_attn_tp_slices`` it reads that slice instead.
"""
global _stack
outer = _stack
stack = _LayerStack(previous_layers, next_layers)
stack = _LayerStack(previous_layers, next_layers, final_read)
_stack = stack
try:
yield stack
@@ -714,6 +727,7 @@ def layer_stack(*, previous_layers=(), next_layers=()):
stack.appends,
previous=_neighbour_stage(stack.previous_layers, last=True),
following=_neighbour_stage(stack.next_layers, last=False),
final_read=stack.final_read,
)
finally:
_stack = outer
@@ -831,7 +845,7 @@ def _detached(declaration):
return replace(declaration, previous=None, prepared_from=None)
def _bind_stack(appends, *, previous, following):
def _bind_stack(appends, *, previous, following, final_read=None):
"""Bind every appended stage, in order, and fill in the boundaries each
append returned."""
if following is not None:
@@ -857,21 +871,77 @@ def _bind_stack(appends, *, previous, following):
sources[id(returned.declaration)] = boundary
bound.append(boundaries)
# The last stage's consumer is the next rank's first stage, if any;
# without one it ends the model's layer stack.
_bind_pending(chain, consumer=following, terminal=following is None)
# without one it ends the model's layer stack, read by the final read.
_bind_pending(
chain,
consumer=following,
terminal=following is None,
final_read=final_read if following is None else None,
)
_check_declared_gathers(appends, bound, remote_producer=previous is not None)
for append, boundaries in zip(appends, bound):
for returned, boundary in zip(append.boundaries, boundaries):
returned.plan = boundary.plan
returned.declaration = boundary.declaration
def _bind_pending(chain: _Chain, *, consumer, terminal):
def _check_declared_gathers(appends, bound, *, remote_producer):
"""A stage that declares its gather over attention TP must have one: at
its own entry, or at its producer's exit. A producer on another pipeline
rank runs it there."""
line = [
(append.origin, boundary)
for append, boundaries in zip(appends, bound)
if append.prepared_from is None
for boundary in boundaries
]
for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]):
if consumer.declaration.attn_tp_gather is None or any(
_gathers_input(path) for path in consumer.plan.paths.values()
):
continue
if producer is None and remote_producer:
continue
if producer is not None and any(
_gathers_output(path) for path in producer.plan.paths.values()
):
continue
error = ValueError(
"an attention-TP gather is declared, but neither this stage's entry "
"nor its producer's exit gathers over attention TP"
)
_note_origin(error, origin)
raise error
def _gathers_input(path):
return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
def _gathers_output(path):
return getattr(path.output_move, "func", None) is update_attn_tp_gather_output
def _ended(declaration, final_read):
"""The stack's last stage as the final read takes it: an FFN on this
rank's attention-TP slice leaves its output there for a read that reads
the slice and gathers what it read."""
if declaration.kind is StageKind.FFN and getattr(
final_read, "reads_attn_tp_slices", False
):
return replace(declaration, exit_rows=ExitRows.SLICE)
return declaration
def _bind_pending(chain: _Chain, *, consumer, terminal, final_read=None):
pending = chain.pending
if pending is None:
return
chain.pending = None
try:
declaration = replace(pending.declaration, terminal=terminal)
if terminal:
declaration = _ended(declaration, final_read)
if pending.predecessor is None:
incoming = _incoming(declaration)
else:
@@ -882,7 +952,12 @@ def _bind_pending(chain: _Chain, *, consumer, terminal):
)
outgoing = _connect(declaration, consumer, residual_from=incoming)
bound = _bind_stage(
declaration, pending.norm, incoming, outgoing, **pending.options
declaration,
pending.norm,
incoming,
outgoing,
final_read=final_read,
**pending.options,
)
except Exception as error:
_note_origin(error, pending.origin)
@@ -70,6 +70,10 @@ class ResidualReadout(Protocol):
is_plain_norm: Read is normalization/optional quantization without changing
the residual, allowing compatible fused add+norm implementations.
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
reads_after_attn_tp_gather: Optional. The read needs every row of the
attention's, so it runs after any gather over attention TP: a
producer that would leave its output on this rank's attention-TP
slice returns it to the attention's rows instead.
completing_fusions: Optional. ReadoutFusion kernels that complete the
sum the input owes with the residual add ahead of this read, tried
before the boundary's own all-reduce or reduce-scatter.
@@ -96,6 +96,9 @@ class AttnBankState:
# A read on this rank's attention-TP slice first tries the bank's kernels
# that also run the collective around it.
fuses_slice_collectives: bool = False
# The bank stays on each rank's attention-TP slice across layers;
# otherwise every rank writes every row and the attention reads them all.
reads_slices: bool = False
def _aggregate(
self, contribution, residual, norm, *, score_proj, score_norm, write
@@ -216,6 +219,10 @@ class _AttnReadout(_BankReadout):
"""The attention input: the bank aggregation and this layer's input norm.
A write layer snapshots the residual this read forms."""
@property
def reads_after_attn_tp_gather(self):
return not self.state.reads_slices
@property
def gathering_reads(self):
state = self.state
@@ -269,20 +276,29 @@ class AttnBankOutputRead:
side's scoring parameters, then the final norm. A callable final norm for
`residual_batch.final_norm`.
A stack that ends on this rank's attention-TP slice of the rows is read
there, against the bank rows this rank wrote, and what it read is
gathered: by the bank's kernel that does both when it takes the batch,
else by ``attn_tp_gather`` when that does, else over the attention-TP
group."""
With ``reads_attn_tp_slices``, the stack may end on this rank's
attention-TP slice of the rows: such a batch is read there, against the
bank rows this rank wrote, and what it read is gathered: by the bank's
kernel that does both when it takes the batch, else by ``attn_tp_gather``
when that does, else over the attention-TP group. Otherwise the stack's
last FFN brings its output to the attention's rows, in ``attn_tp_gather``
when that takes the batch."""
def __init__(
self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None
self,
bank: AttnBank,
score_proj,
score_norm,
norm,
attn_tp_gather=None,
reads_attn_tp_slices=False,
):
self.bank = bank
self.score_proj = score_proj
self.score_norm = score_norm
self.norm = norm
self.attn_tp_gather = attn_tp_gather
self.reads_attn_tp_slices = reads_attn_tp_slices
def __call__(self, hidden_states, residual=None):
bank = self.bank.require()
@@ -44,6 +44,8 @@ class GatedResidualState:
ffn_mix: The FFN stage's read.
attn_combine: Writes the attention output back into the streams.
ffn_combine: Writes the FFN output back into the streams.
attn_reads_every_row: Whether attn_mix needs every row of the
attention's, as a contribution computed for all of them does.
"""
expand: Callable
@@ -51,6 +53,7 @@ class GatedResidualState:
ffn_mix: Callable
attn_combine: Callable
ffn_combine: Callable
attn_reads_every_row: bool = False
# Produced by a stage's read and consumed by that same stage's write-back.
normed: Optional[torch.Tensor] = None
@@ -110,6 +113,10 @@ class _AttnReadout:
def __init__(self, state: GatedResidualState):
self.state = state
@property
def reads_after_attn_tp_gather(self):
return self.state.attn_reads_every_row
def init_residual(self, hidden_states):
return self.state.expand(hidden_states)
File diff suppressed because it is too large Load Diff
+3 -11
View File
@@ -37,7 +37,6 @@ from sglang.srt.layers.hyperconnection import (
HyperConnectionConfig,
)
from sglang.srt.layers.layer_boundary import (
ExitRows,
GatedResidualState,
append_stages,
declare_attn,
@@ -1434,13 +1433,7 @@ def _has_ple(layer_id, config) -> bool:
return (layer_id + 1) in config.ple_layer_ids
def _ffn_exit_rows(layer_id, config):
"""A PLE layer's read adds an embedding computed for every row, so the FFN
before it hands its output on as full attention rows."""
return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None
def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
def _build_qwen4_exp_stages(residual, *, sparse):
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
Each read normalizes the streams itself, so neither stage binds a norm, and
@@ -1458,7 +1451,6 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
next_layer_sparse=sparse,
read=residual.ffn_readout,
update=residual.ffn_update,
exit_rows=_ffn_exit_rows(layer_id, config),
),
None,
),
@@ -1526,10 +1518,10 @@ class Qwen4ExpLayerExtensionMixin:
ffn_mix=self.mlp_hyper_connection.mix,
attn_combine=self.attn_hyper_connection.combine,
ffn_combine=self.mlp_hyper_connection.combine,
# The PLE embedding is computed for every row.
attn_reads_every_row=self.ple is not None,
).residual_ops(),
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
layer_id=layer_id,
config=config,
)
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
+6 -1
View File
@@ -1590,13 +1590,15 @@ def make_layers(
prefix: str = "",
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
final_read: Optional[Any] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make a list of layers with the given layer function.
The local layers are built inside one layer stack, so layers that declare
stage boundaries connect in order without naming their neighbours. Across
a pipeline stage boundary the stack learns the neighbouring stage from the
layer itself, built again on the meta device.
layer itself, built again on the meta device. ``final_read`` is the
stack's terminal read when it is not a plain final norm (see layer_stack).
"""
# circular imports
from sglang.srt.distributed import get_pp_indices
@@ -1623,6 +1625,7 @@ def make_layers(
with layer_stack(
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
final_read=final_read,
):
modules = torch.nn.ModuleList(
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
@@ -1649,6 +1652,7 @@ def make_pp_layers(
prefix: str = "",
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
final_read: Optional[Any] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make this pipeline stage's layers, and return them with the stage's range.
@@ -1663,6 +1667,7 @@ def make_pp_layers(
prefix=prefix,
return_tuple=return_tuple,
offloader_kwargs=offloader_kwargs,
final_read=final_read,
)