[Refactor] Remove Kimi K3's own layer communication and let the consumer set an FFN's exit (#42936)
This commit is contained in:
@@ -47,10 +47,6 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_tp_gather_input,
|
||||
update_attn_tp_gather_output,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_input_default,
|
||||
_attn_input_scattered,
|
||||
@@ -191,6 +187,7 @@ class StagePlan:
|
||||
qkv_latent_func=None,
|
||||
fusions=None,
|
||||
attn_tp_gather=None,
|
||||
exit_gather=None,
|
||||
):
|
||||
self.norm = norm
|
||||
self.edges = dict(variants)
|
||||
@@ -242,7 +239,7 @@ class StagePlan:
|
||||
else bind_exit(
|
||||
edges.outgoing,
|
||||
cp_moves=edges.cp_moves,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
attn_tp_gather=exit_gather,
|
||||
)
|
||||
)
|
||||
self.paths[variant] = StagePath(
|
||||
@@ -253,15 +250,6 @@ class StagePlan:
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
if attn_tp_gather is not None and not any(
|
||||
getattr(path.output_move, "func", None) is update_attn_tp_gather_output
|
||||
or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
|
||||
for path in self.paths.values()
|
||||
):
|
||||
raise ValueError(
|
||||
"an attention-TP gather is declared, but no batch variant "
|
||||
"gathers over attention TP at this stage's boundaries"
|
||||
)
|
||||
|
||||
@property
|
||||
def incoming_residual_rows(self):
|
||||
@@ -319,7 +307,7 @@ class StagePlan:
|
||||
)
|
||||
|
||||
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
@@ -354,6 +342,13 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
terminal=declaration.terminal,
|
||||
writes_at_handoff=declaration.writes_at_handoff,
|
||||
attn_tp_gather=declaration.attn_tp_gather,
|
||||
# The exit runs its consumer's gather: the next stage's, or the
|
||||
# final read's.
|
||||
exit_gather=getattr(
|
||||
outgoing.consumer if outgoing.consumer is not None else final_read,
|
||||
"attn_tp_gather",
|
||||
None,
|
||||
),
|
||||
finishes_directly=declaration.kind is StageKind.ATTENTION
|
||||
and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL,
|
||||
**options,
|
||||
|
||||
@@ -45,9 +45,9 @@ class ExitRows(Enum):
|
||||
|
||||
ATTENTION = auto()
|
||||
TBO_SPLIT = auto()
|
||||
# The rows the FFN ran on, also at the stack's end: an FFN on this rank's
|
||||
# attention-TP slice leaves its output there, and the model's final read
|
||||
# must read that slice and gather it.
|
||||
# The rows the FFN ran on, also at the stack's end: the layer stack's last
|
||||
# FFN, when the model's final read reads this rank's attention-TP slice
|
||||
# and gathers it.
|
||||
SLICE = auto()
|
||||
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_tp_gather_input,
|
||||
update_attn_tp_gather_output,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
@@ -261,10 +265,14 @@ class StageDeclaration:
|
||||
sum, so the exit owes none. For an FFN whose output sum is fused
|
||||
with a reduction its computation needs anyway.
|
||||
attn_tp_gather: Optional implementation of the gather over attention
|
||||
TP this stage's boundaries run: given this rank's contiguous slice
|
||||
of the rows, it returns them all, in rank order, or None to leave
|
||||
the gather to the boundary. Called on every batch, so it must be
|
||||
CUDA-graph safe.
|
||||
TP that brings rows into this stage: at its own entry, or at its
|
||||
producer's exit, which runs its consumer's gather. Given this rank's
|
||||
contiguous slice of the rows, it returns them all, in rank order,
|
||||
or None to leave the gather to the boundary. Called on every batch,
|
||||
so it must be CUDA-graph safe. Across a pipeline boundary the
|
||||
producer's rank takes it from the neighbouring layer it builds on
|
||||
the meta device, so it may use only the communication state of the
|
||||
rank it runs on, not the declaring layer's weights or buffers.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
@@ -346,8 +354,8 @@ def declare_attn(
|
||||
stage's input runs it, so no fused add + norm takes that input.
|
||||
Requires ALWAYS_PARTIAL.
|
||||
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
|
||||
attn_tp_gather: Implementation of this stage's gathers over attention
|
||||
TP, tried before the boundary's own (see StageDeclaration).
|
||||
attn_tp_gather: Implementation of the gather over attention TP into
|
||||
this stage, tried before the boundary's own (see StageDeclaration).
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -379,7 +387,6 @@ def declare_ffn(
|
||||
dense_tp_size=None,
|
||||
exit_rows=None,
|
||||
output_complete=False,
|
||||
attn_tp_gather=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
@@ -396,8 +403,6 @@ def declare_ffn(
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
fused with a reduction it needs anyway; the exit then owes none.
|
||||
attn_tp_gather: Implementation of this stage's gathers over attention
|
||||
TP, tried before the boundary's own (see StageDeclaration).
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -411,7 +416,6 @@ def declare_ffn(
|
||||
dense_tp_size=dense_tp_size,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
output_complete=output_complete,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
)
|
||||
|
||||
|
||||
@@ -438,7 +442,10 @@ def _resolve_stage(stage, variant, following=None):
|
||||
output_complete=stage.output_complete,
|
||||
)
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION:
|
||||
if exit_rows is ExitRows.ATTENTION or (
|
||||
following is not None
|
||||
and getattr(following.read, "reads_after_attn_tp_gather", False)
|
||||
):
|
||||
returned = attention
|
||||
elif exit_rows is ExitRows.SLICE:
|
||||
# The residual's rows during the FFN: this rank's slice for an FFN
|
||||
@@ -674,12 +681,13 @@ class _LayerStack:
|
||||
"""A layer stack under construction: what was appended, in order, and how
|
||||
to reach the layers other pipeline ranks hold on either side of it."""
|
||||
|
||||
__slots__ = ("appends", "previous_layers", "next_layers")
|
||||
__slots__ = ("appends", "previous_layers", "next_layers", "final_read")
|
||||
|
||||
def __init__(self, previous_layers=(), next_layers=()):
|
||||
def __init__(self, previous_layers=(), next_layers=(), final_read=None):
|
||||
self.appends = []
|
||||
self.previous_layers = previous_layers
|
||||
self.next_layers = next_layers
|
||||
self.final_read = final_read
|
||||
|
||||
|
||||
# The stack being built; layer_stack saves and restores an outer one.
|
||||
@@ -687,7 +695,7 @@ _stack: Optional[_LayerStack] = None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def layer_stack(*, previous_layers=(), next_layers=()):
|
||||
def layer_stack(*, previous_layers=(), next_layers=(), final_read=None):
|
||||
"""Open a layer stack that append_stages extends in order.
|
||||
|
||||
Every stage binds when the stack closes, once its producer and its
|
||||
@@ -702,10 +710,15 @@ def layer_stack(*, previous_layers=(), next_layers=()):
|
||||
stack's first. What they build is discarded.
|
||||
next_layers: Likewise for the layers after this stack, whose first
|
||||
declared stage is the consumer of this stack's last.
|
||||
final_read: The model's final read of the stack's output, when it is
|
||||
not a plain final norm: the consumer of the last stage when no
|
||||
later layer declares one. Its ``attn_tp_gather`` gathers the rows
|
||||
that stage leaves on this rank's attention-TP slice, and with
|
||||
``reads_attn_tp_slices`` it reads that slice instead.
|
||||
"""
|
||||
global _stack
|
||||
outer = _stack
|
||||
stack = _LayerStack(previous_layers, next_layers)
|
||||
stack = _LayerStack(previous_layers, next_layers, final_read)
|
||||
_stack = stack
|
||||
try:
|
||||
yield stack
|
||||
@@ -714,6 +727,7 @@ def layer_stack(*, previous_layers=(), next_layers=()):
|
||||
stack.appends,
|
||||
previous=_neighbour_stage(stack.previous_layers, last=True),
|
||||
following=_neighbour_stage(stack.next_layers, last=False),
|
||||
final_read=stack.final_read,
|
||||
)
|
||||
finally:
|
||||
_stack = outer
|
||||
@@ -831,7 +845,7 @@ def _detached(declaration):
|
||||
return replace(declaration, previous=None, prepared_from=None)
|
||||
|
||||
|
||||
def _bind_stack(appends, *, previous, following):
|
||||
def _bind_stack(appends, *, previous, following, final_read=None):
|
||||
"""Bind every appended stage, in order, and fill in the boundaries each
|
||||
append returned."""
|
||||
if following is not None:
|
||||
@@ -857,21 +871,77 @@ def _bind_stack(appends, *, previous, following):
|
||||
sources[id(returned.declaration)] = boundary
|
||||
bound.append(boundaries)
|
||||
# The last stage's consumer is the next rank's first stage, if any;
|
||||
# without one it ends the model's layer stack.
|
||||
_bind_pending(chain, consumer=following, terminal=following is None)
|
||||
# without one it ends the model's layer stack, read by the final read.
|
||||
_bind_pending(
|
||||
chain,
|
||||
consumer=following,
|
||||
terminal=following is None,
|
||||
final_read=final_read if following is None else None,
|
||||
)
|
||||
_check_declared_gathers(appends, bound, remote_producer=previous is not None)
|
||||
for append, boundaries in zip(appends, bound):
|
||||
for returned, boundary in zip(append.boundaries, boundaries):
|
||||
returned.plan = boundary.plan
|
||||
returned.declaration = boundary.declaration
|
||||
|
||||
|
||||
def _bind_pending(chain: _Chain, *, consumer, terminal):
|
||||
def _check_declared_gathers(appends, bound, *, remote_producer):
|
||||
"""A stage that declares its gather over attention TP must have one: at
|
||||
its own entry, or at its producer's exit. A producer on another pipeline
|
||||
rank runs it there."""
|
||||
line = [
|
||||
(append.origin, boundary)
|
||||
for append, boundaries in zip(appends, bound)
|
||||
if append.prepared_from is None
|
||||
for boundary in boundaries
|
||||
]
|
||||
for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]):
|
||||
if consumer.declaration.attn_tp_gather is None or any(
|
||||
_gathers_input(path) for path in consumer.plan.paths.values()
|
||||
):
|
||||
continue
|
||||
if producer is None and remote_producer:
|
||||
continue
|
||||
if producer is not None and any(
|
||||
_gathers_output(path) for path in producer.plan.paths.values()
|
||||
):
|
||||
continue
|
||||
error = ValueError(
|
||||
"an attention-TP gather is declared, but neither this stage's entry "
|
||||
"nor its producer's exit gathers over attention TP"
|
||||
)
|
||||
_note_origin(error, origin)
|
||||
raise error
|
||||
|
||||
|
||||
def _gathers_input(path):
|
||||
return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input
|
||||
|
||||
|
||||
def _gathers_output(path):
|
||||
return getattr(path.output_move, "func", None) is update_attn_tp_gather_output
|
||||
|
||||
|
||||
def _ended(declaration, final_read):
|
||||
"""The stack's last stage as the final read takes it: an FFN on this
|
||||
rank's attention-TP slice leaves its output there for a read that reads
|
||||
the slice and gathers what it read."""
|
||||
if declaration.kind is StageKind.FFN and getattr(
|
||||
final_read, "reads_attn_tp_slices", False
|
||||
):
|
||||
return replace(declaration, exit_rows=ExitRows.SLICE)
|
||||
return declaration
|
||||
|
||||
|
||||
def _bind_pending(chain: _Chain, *, consumer, terminal, final_read=None):
|
||||
pending = chain.pending
|
||||
if pending is None:
|
||||
return
|
||||
chain.pending = None
|
||||
try:
|
||||
declaration = replace(pending.declaration, terminal=terminal)
|
||||
if terminal:
|
||||
declaration = _ended(declaration, final_read)
|
||||
if pending.predecessor is None:
|
||||
incoming = _incoming(declaration)
|
||||
else:
|
||||
@@ -882,7 +952,12 @@ def _bind_pending(chain: _Chain, *, consumer, terminal):
|
||||
)
|
||||
outgoing = _connect(declaration, consumer, residual_from=incoming)
|
||||
bound = _bind_stage(
|
||||
declaration, pending.norm, incoming, outgoing, **pending.options
|
||||
declaration,
|
||||
pending.norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
final_read=final_read,
|
||||
**pending.options,
|
||||
)
|
||||
except Exception as error:
|
||||
_note_origin(error, pending.origin)
|
||||
|
||||
@@ -70,6 +70,10 @@ class ResidualReadout(Protocol):
|
||||
is_plain_norm: Read is normalization/optional quantization without changing
|
||||
the residual, allowing compatible fused add+norm implementations.
|
||||
reads_before_dp_gather: Preserve this read on source rows before a DP gather.
|
||||
reads_after_attn_tp_gather: Optional. The read needs every row of the
|
||||
attention's, so it runs after any gather over attention TP: a
|
||||
producer that would leave its output on this rank's attention-TP
|
||||
slice returns it to the attention's rows instead.
|
||||
completing_fusions: Optional. ReadoutFusion kernels that complete the
|
||||
sum the input owes with the residual add ahead of this read, tried
|
||||
before the boundary's own all-reduce or reduce-scatter.
|
||||
|
||||
@@ -96,6 +96,9 @@ class AttnBankState:
|
||||
# A read on this rank's attention-TP slice first tries the bank's kernels
|
||||
# that also run the collective around it.
|
||||
fuses_slice_collectives: bool = False
|
||||
# The bank stays on each rank's attention-TP slice across layers;
|
||||
# otherwise every rank writes every row and the attention reads them all.
|
||||
reads_slices: bool = False
|
||||
|
||||
def _aggregate(
|
||||
self, contribution, residual, norm, *, score_proj, score_norm, write
|
||||
@@ -216,6 +219,10 @@ class _AttnReadout(_BankReadout):
|
||||
"""The attention input: the bank aggregation and this layer's input norm.
|
||||
A write layer snapshots the residual this read forms."""
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
return not self.state.reads_slices
|
||||
|
||||
@property
|
||||
def gathering_reads(self):
|
||||
state = self.state
|
||||
@@ -269,20 +276,29 @@ class AttnBankOutputRead:
|
||||
side's scoring parameters, then the final norm. A callable final norm for
|
||||
`residual_batch.final_norm`.
|
||||
|
||||
A stack that ends on this rank's attention-TP slice of the rows is read
|
||||
there, against the bank rows this rank wrote, and what it read is
|
||||
gathered: by the bank's kernel that does both when it takes the batch,
|
||||
else by ``attn_tp_gather`` when that does, else over the attention-TP
|
||||
group."""
|
||||
With ``reads_attn_tp_slices``, the stack may end on this rank's
|
||||
attention-TP slice of the rows: such a batch is read there, against the
|
||||
bank rows this rank wrote, and what it read is gathered: by the bank's
|
||||
kernel that does both when it takes the batch, else by ``attn_tp_gather``
|
||||
when that does, else over the attention-TP group. Otherwise the stack's
|
||||
last FFN brings its output to the attention's rows, in ``attn_tp_gather``
|
||||
when that takes the batch."""
|
||||
|
||||
def __init__(
|
||||
self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None
|
||||
self,
|
||||
bank: AttnBank,
|
||||
score_proj,
|
||||
score_norm,
|
||||
norm,
|
||||
attn_tp_gather=None,
|
||||
reads_attn_tp_slices=False,
|
||||
):
|
||||
self.bank = bank
|
||||
self.score_proj = score_proj
|
||||
self.score_norm = score_norm
|
||||
self.norm = norm
|
||||
self.attn_tp_gather = attn_tp_gather
|
||||
self.reads_attn_tp_slices = reads_attn_tp_slices
|
||||
|
||||
def __call__(self, hidden_states, residual=None):
|
||||
bank = self.bank.require()
|
||||
|
||||
@@ -44,6 +44,8 @@ class GatedResidualState:
|
||||
ffn_mix: The FFN stage's read.
|
||||
attn_combine: Writes the attention output back into the streams.
|
||||
ffn_combine: Writes the FFN output back into the streams.
|
||||
attn_reads_every_row: Whether attn_mix needs every row of the
|
||||
attention's, as a contribution computed for all of them does.
|
||||
"""
|
||||
|
||||
expand: Callable
|
||||
@@ -51,6 +53,7 @@ class GatedResidualState:
|
||||
ffn_mix: Callable
|
||||
attn_combine: Callable
|
||||
ffn_combine: Callable
|
||||
attn_reads_every_row: bool = False
|
||||
# Produced by a stage's read and consumed by that same stage's write-back.
|
||||
normed: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -110,6 +113,10 @@ class _AttnReadout:
|
||||
def __init__(self, state: GatedResidualState):
|
||||
self.state = state
|
||||
|
||||
@property
|
||||
def reads_after_attn_tp_gather(self):
|
||||
return self.state.attn_reads_every_row
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return self.state.expand(hidden_states)
|
||||
|
||||
|
||||
+104
-562
File diff suppressed because it is too large
Load Diff
@@ -37,7 +37,6 @@ from sglang.srt.layers.hyperconnection import (
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
GatedResidualState,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -1434,13 +1433,7 @@ def _has_ple(layer_id, config) -> bool:
|
||||
return (layer_id + 1) in config.ple_layer_ids
|
||||
|
||||
|
||||
def _ffn_exit_rows(layer_id, config):
|
||||
"""A PLE layer's read adds an embedding computed for every row, so the FFN
|
||||
before it hands its output on as full attention rows."""
|
||||
return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None
|
||||
|
||||
|
||||
def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
|
||||
def _build_qwen4_exp_stages(residual, *, sparse):
|
||||
"""The attention and FFN stage boundaries of one gated hyper-connection layer.
|
||||
|
||||
Each read normalizes the streams itself, so neither stage binds a norm, and
|
||||
@@ -1458,7 +1451,6 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config):
|
||||
next_layer_sparse=sparse,
|
||||
read=residual.ffn_readout,
|
||||
update=residual.ffn_update,
|
||||
exit_rows=_ffn_exit_rows(layer_id, config),
|
||||
),
|
||||
None,
|
||||
),
|
||||
@@ -1526,10 +1518,10 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
ffn_mix=self.mlp_hyper_connection.mix,
|
||||
attn_combine=self.attn_hyper_connection.combine,
|
||||
ffn_combine=self.mlp_hyper_connection.combine,
|
||||
# The PLE embedding is computed for every row.
|
||||
attn_reads_every_row=self.ple is not None,
|
||||
).residual_ops(),
|
||||
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
)
|
||||
|
||||
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -1590,13 +1590,15 @@ def make_layers(
|
||||
prefix: str = "",
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
final_read: Optional[Any] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make a list of layers with the given layer function.
|
||||
|
||||
The local layers are built inside one layer stack, so layers that declare
|
||||
stage boundaries connect in order without naming their neighbours. Across
|
||||
a pipeline stage boundary the stack learns the neighbouring stage from the
|
||||
layer itself, built again on the meta device.
|
||||
layer itself, built again on the meta device. ``final_read`` is the
|
||||
stack's terminal read when it is not a plain final norm (see layer_stack).
|
||||
"""
|
||||
# circular imports
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
@@ -1623,6 +1625,7 @@ def make_layers(
|
||||
with layer_stack(
|
||||
previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))],
|
||||
next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)],
|
||||
final_read=final_read,
|
||||
):
|
||||
modules = torch.nn.ModuleList(
|
||||
[PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)]
|
||||
@@ -1649,6 +1652,7 @@ def make_pp_layers(
|
||||
prefix: str = "",
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
final_read: Optional[Any] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make this pipeline stage's layers, and return them with the stage's range.
|
||||
|
||||
@@ -1663,6 +1667,7 @@ def make_pp_layers(
|
||||
prefix=prefix,
|
||||
return_tuple=return_tuple,
|
||||
offloader_kwargs=offloader_kwargs,
|
||||
final_read=final_read,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user