From dc0b394471be5bb0738ea43d9ff24fe0aad2dba9 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:27:46 -0700 Subject: [PATCH] [Refactor] Remove Kimi K3's own layer communication and let the consumer set an FFN's exit (#42936) --- docs/docs/developer_guide/layer_boundary.mdx | 5 +- .../srt/layers/layer_boundary/construction.py | 25 +- .../srt/layers/layer_boundary/contracts.py | 6 +- .../srt/layers/layer_boundary/factories.py | 115 ++- .../layer_boundary/residual/__init__.py | 4 + .../layer_boundary/residual/attn_bank.py | 28 +- .../layers/layer_boundary/residual/gated.py | 7 + python/sglang/srt/models/kimi_k3.py | 666 +++--------------- python/sglang/srt/models/qwen4_exp.py | 14 +- python/sglang/srt/utils/common.py | 7 +- .../unit/layer_boundary/test_append_stages.py | 86 ++- .../test_attn_bank_residual_ops.py | 41 +- .../test_kimi_k3_stage_kernels.py | 26 +- .../layer_boundary/test_qwen4_exp_ple_rows.py | 8 +- .../test_kimi_attention_parallel_groups.py | 5 +- .../layers/test_kimi_mlp_parallel_groups.py | 111 ++- 16 files changed, 424 insertions(+), 730 deletions(-) diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index eb243cbbe6..e3269c9d50 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -82,6 +82,7 @@ The producer owns `ResidualUpdate`; the consumer owns `ResidualReadout` and its - A nonlinear update cannot generally move before reduction. The post operation of manifold-constrained hyper-connections (MHC) is one such case. - `applied_at_exit` requests an update at the producer exit. A deferred update must guarantee that its parameters and state outlive that producer. - `ResidualReadout.reads_before_dp_gather` preserves reads that must execute on the source rows before a DP gather. +- A read that needs every row of the attention's, such as one that adds a contribution computed for every row, declares `reads_after_attn_tp_gather`. A producer that would leave its output on this rank's attention-TP slice then returns it to the attention's rows; Qwen4-exp's per-layer embedding read and Kimi K3's attention-residual bank declare it. A read returns `(compute_input, residual)`. `update_and_read()` receives the producer's update together with the read, so an implementation can fuse them: the plain add+norm read does, and keeps its FP32 accumulation instead of normalizing a separately rounded snapshot. Fusion is not guaranteed. An implementation such as MHC may run the update and the read as separate steps, provided it preserves its existing rounding order. @@ -202,6 +203,8 @@ hidden_states = residual_batch.final_norm( `skip_empty=True` still completes owed work on an empty batch, which may be a collective that other ranks join, and skips only the norm kernel. `final_norm()` and `to_pp()` release the batch's stream after they hand on its output. +When the final read is more than a norm, pass it to `make_layers(final_read=...)` (or `layer_stack(final_read=...)`): it is then the last stage's consumer. Its `attn_tp_gather` gathers the rows the last FFN leaves on this rank's attention-TP slice, and with `reads_attn_tp_slices` it reads that slice itself and the last FFN leaves its output there. Kimi K3's `AttnBankOutputRead` is such a read. + This is a non-pipeline stack fragment. Pipeline reception uses the first boundary's `from_pp(tensors, forward_batch)` to reconstruct a stream; the sender uses `residual_batch.to_pp()`. A stream the producer already wrote travels as `hidden_states` alone and arrives written: MHC's expanded streams, and the output of an FFN on its own rows (its attention-TP slice, or an unpadded batch's rows), which writes it at its exit before a pipeline handoff. The receiver knows this from the producer's declaration, not from which tensors arrive, since a captured CUDA graph's input buffers hold every key. Dynamic completion work is resolved before transport. By default, `to_pp()` preserves a statically declared partial sum for the receiver's incoming contract to reconstruct and complete. Do not call `complete_output()` before that handoff: it would complete a sum that the receiver still expects to perform. A terminal FFN normally completes its output before the final norm. The exception is a MoE finalize handoff. When the stage's fusion provider can defer the finalize at the terminal layer (for CuteDSL, `install_cutedsl_fusion(..., terminal_finalize=True)`), the exit may leave a `DeferredFinalize` for the final norm. Pass the handle straight to `residual_batch.final_norm(hidden_states, forward_batch, self.norm, finalize_norm=service)`, where `service` is the object `install_cutedsl_fusion()` returned; its `finalize(handoff, residual, gamma)` fuses finalize, all-reduce, residual add, and norm. The fused path reads `layernorm.gemma_weight`, so it currently expects a `GemmaRMSNorm` final norm. Without `finalize_norm`, `final_norm()` first completes the handoff unfused, and `capture` is then allowed. With `finalize_norm`, `final_norm()` rejects `capture` when a handoff arrives. Calling `complete_output()` first runs the producer's unfused finalize, and `snapshot()` raises on a handoff. See `python/sglang/srt/models/qwen3_5.py`. @@ -339,7 +342,7 @@ A backend fusion provider is passed per stage as the `fusions` option, `(declara `install_cutedsl_fusion()` and `prepare_cutedsl_fusion()` find providers through the decoder layer's `attn_boundary` and `ffn_boundary` attributes. -Consumer fusion candidates are ordered. A candidate returning `None` must do so before modifying inputs or starting a collective, so fallback remains valid. `FfnInputFusion` declares the sum group it completes with residual add and norm. A residual read may supply its own candidates as `completing_fusions`: each `ReadoutFusion` completes a sum with the residual add, ahead of the read, and is tried before the built-in collective; one with `reads=True` also does the read. Where a stage reads its input on this rank's attention-TP slice and then gathers it, a read's `gathering_reads` may do both in one kernel. A stage may declare its own implementation of its gathers over attention TP (`declare_attn(attn_tp_gather=...)`, `declare_ffn(attn_tp_gather=...)`), tried on every batch before the built-in gather. The consumer only tries compatible candidates for the declared update/read capabilities. +Consumer fusion candidates are ordered. A candidate returning `None` must do so before modifying inputs or starting a collective, so fallback remains valid. `FfnInputFusion` declares the sum group it completes with residual add and norm. A residual read may supply its own candidates as `completing_fusions`: each `ReadoutFusion` completes a sum with the residual add, ahead of the read, and is tried before the built-in collective; one with `reads=True` also does the read. Where a stage reads its input on this rank's attention-TP slice and then gathers it, a read's `gathering_reads` may do both in one kernel. A stage may declare its own implementation of the gather over attention TP that brings rows into it (`declare_attn(attn_tp_gather=...)`), tried on every batch before the built-in gather: at its own entry, or at its producer's exit, since an exit runs its consumer's gather. Across a pipeline boundary the producer's rank takes that gather from the neighbouring layer it builds on the meta device, so the gather may use only the communication state of the rank it runs on, not the declaring layer's weights or buffers. The consumer only tries compatible candidates for the declared update/read capabilities. An `OutputTransform` declares contribution processing before residual update. Its explicit `before_reduce_scatter` setting preserves a supported implementation's order; it is not permission to commute arbitrary operations across a reduction. diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 34a7e1f854..f426ec14a6 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -47,10 +47,6 @@ from sglang.srt.layers.layer_boundary.layout import ( batches_are_unpadded, is_dense_ffn_fully_dp, ) -from sglang.srt.layers.layer_boundary.ops import ( - attn_tp_gather_input, - update_attn_tp_gather_output, -) from sglang.srt.layers.layer_boundary.prepare import ( _attn_input_default, _attn_input_scattered, @@ -191,6 +187,7 @@ class StagePlan: qkv_latent_func=None, fusions=None, attn_tp_gather=None, + exit_gather=None, ): self.norm = norm self.edges = dict(variants) @@ -242,7 +239,7 @@ class StagePlan: else bind_exit( edges.outgoing, cp_moves=edges.cp_moves, - attn_tp_gather=attn_tp_gather, + attn_tp_gather=exit_gather, ) ) self.paths[variant] = StagePath( @@ -253,15 +250,6 @@ class StagePlan: returns_over_dp=out.returns_over_dp, writes_at_handoff=writes_at_handoff, ) - if attn_tp_gather is not None and not any( - getattr(path.output_move, "func", None) is update_attn_tp_gather_output - or getattr(path.entry.input_move, "func", None) is attn_tp_gather_input - for path in self.paths.values() - ): - raise ValueError( - "an attention-TP gather is declared, but no batch variant " - "gathers over attention TP at this stage's boundaries" - ) @property def incoming_residual_rows(self): @@ -319,7 +307,7 @@ class StagePlan: ) -def _bind_stage(declaration, norm, incoming, outgoing, **options): +def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options): if incoming.consumer != declaration or outgoing.producer != declaration: raise ValueError("connections do not match the stage declaration") if incoming.entries.keys() != outgoing.exits.keys(): @@ -354,6 +342,13 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options): terminal=declaration.terminal, writes_at_handoff=declaration.writes_at_handoff, attn_tp_gather=declaration.attn_tp_gather, + # The exit runs its consumer's gather: the next stage's, or the + # final read's. + exit_gather=getattr( + outgoing.consumer if outgoing.consumer is not None else final_read, + "attn_tp_gather", + None, + ), finishes_directly=declaration.kind is StageKind.ATTENTION and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL, **options, diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 2e65f79aae..6ca117618c 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -45,9 +45,9 @@ class ExitRows(Enum): ATTENTION = auto() TBO_SPLIT = auto() - # The rows the FFN ran on, also at the stack's end: an FFN on this rank's - # attention-TP slice leaves its output there, and the model's final read - # must read that slice and gather it. + # The rows the FFN ran on, also at the stack's end: the layer stack's last + # FFN, when the model's final read reads this rank's attention-TP slice + # and gathers it. SLICE = auto() diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 988fa4f4f9..fed64cf858 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -44,6 +44,10 @@ from sglang.srt.layers.layer_boundary.layout import ( moe_gathers_over_moe_cp, token_axis_sizes, ) +from sglang.srt.layers.layer_boundary.ops import ( + attn_tp_gather_input, + update_attn_tp_gather_output, +) from sglang.srt.layers.layer_boundary.output import OutputTransform from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate from sglang.srt.layers.layer_boundary.residual.add_norm import ( @@ -261,10 +265,14 @@ class StageDeclaration: sum, so the exit owes none. For an FFN whose output sum is fused with a reduction its computation needs anyway. attn_tp_gather: Optional implementation of the gather over attention - TP this stage's boundaries run: given this rank's contiguous slice - of the rows, it returns them all, in rank order, or None to leave - the gather to the boundary. Called on every batch, so it must be - CUDA-graph safe. + TP that brings rows into this stage: at its own entry, or at its + producer's exit, which runs its consumer's gather. Given this rank's + contiguous slice of the rows, it returns them all, in rank order, + or None to leave the gather to the boundary. Called on every batch, + so it must be CUDA-graph safe. Across a pipeline boundary the + producer's rank takes it from the neighbouring layer it builds on + the meta device, so it may use only the communication state of the + rank it runs on, not the declaring layer's weights or buffers. previous: Declaration whose output this stage consumes, as the stack records it; across pipeline ranks it is built locally. prepared_from: Declaration whose already-read input a branch reuses. @@ -346,8 +354,8 @@ def declare_attn( stage's input runs it, so no fused add + norm takes that input. Requires ALWAYS_PARTIAL. tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers. - attn_tp_gather: Implementation of this stage's gathers over attention - TP, tried before the boundary's own (see StageDeclaration). + attn_tp_gather: Implementation of the gather over attention TP into + this stage, tried before the boundary's own (see StageDeclaration). Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -379,7 +387,6 @@ def declare_ffn( dense_tp_size=None, exit_rows=None, output_complete=False, - attn_tp_gather=None, ): """Declare a dense or MoE FFN independently of its compute module. @@ -396,8 +403,6 @@ def declare_ffn( the adjacent FFN kinds and TBO configuration. output_complete: Whether the compute completes its own output sum, fused with a reduction it needs anyway; the exit then owes none. - attn_tp_gather: Implementation of this stage's gathers over attention - TP, tried before the boundary's own (see StageDeclaration). Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -411,7 +416,6 @@ def declare_ffn( dense_tp_size=dense_tp_size, exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), output_complete=output_complete, - attn_tp_gather=attn_tp_gather, ) @@ -438,7 +442,10 @@ def _resolve_stage(stage, variant, following=None): output_complete=stage.output_complete, ) exit_rows = resolve_exit_rows(stage.exit_rows) - if exit_rows is ExitRows.ATTENTION: + if exit_rows is ExitRows.ATTENTION or ( + following is not None + and getattr(following.read, "reads_after_attn_tp_gather", False) + ): returned = attention elif exit_rows is ExitRows.SLICE: # The residual's rows during the FFN: this rank's slice for an FFN @@ -674,12 +681,13 @@ class _LayerStack: """A layer stack under construction: what was appended, in order, and how to reach the layers other pipeline ranks hold on either side of it.""" - __slots__ = ("appends", "previous_layers", "next_layers") + __slots__ = ("appends", "previous_layers", "next_layers", "final_read") - def __init__(self, previous_layers=(), next_layers=()): + def __init__(self, previous_layers=(), next_layers=(), final_read=None): self.appends = [] self.previous_layers = previous_layers self.next_layers = next_layers + self.final_read = final_read # The stack being built; layer_stack saves and restores an outer one. @@ -687,7 +695,7 @@ _stack: Optional[_LayerStack] = None @contextlib.contextmanager -def layer_stack(*, previous_layers=(), next_layers=()): +def layer_stack(*, previous_layers=(), next_layers=(), final_read=None): """Open a layer stack that append_stages extends in order. Every stage binds when the stack closes, once its producer and its @@ -702,10 +710,15 @@ def layer_stack(*, previous_layers=(), next_layers=()): stack's first. What they build is discarded. next_layers: Likewise for the layers after this stack, whose first declared stage is the consumer of this stack's last. + final_read: The model's final read of the stack's output, when it is + not a plain final norm: the consumer of the last stage when no + later layer declares one. Its ``attn_tp_gather`` gathers the rows + that stage leaves on this rank's attention-TP slice, and with + ``reads_attn_tp_slices`` it reads that slice instead. """ global _stack outer = _stack - stack = _LayerStack(previous_layers, next_layers) + stack = _LayerStack(previous_layers, next_layers, final_read) _stack = stack try: yield stack @@ -714,6 +727,7 @@ def layer_stack(*, previous_layers=(), next_layers=()): stack.appends, previous=_neighbour_stage(stack.previous_layers, last=True), following=_neighbour_stage(stack.next_layers, last=False), + final_read=stack.final_read, ) finally: _stack = outer @@ -831,7 +845,7 @@ def _detached(declaration): return replace(declaration, previous=None, prepared_from=None) -def _bind_stack(appends, *, previous, following): +def _bind_stack(appends, *, previous, following, final_read=None): """Bind every appended stage, in order, and fill in the boundaries each append returned.""" if following is not None: @@ -857,21 +871,77 @@ def _bind_stack(appends, *, previous, following): sources[id(returned.declaration)] = boundary bound.append(boundaries) # The last stage's consumer is the next rank's first stage, if any; - # without one it ends the model's layer stack. - _bind_pending(chain, consumer=following, terminal=following is None) + # without one it ends the model's layer stack, read by the final read. + _bind_pending( + chain, + consumer=following, + terminal=following is None, + final_read=final_read if following is None else None, + ) + _check_declared_gathers(appends, bound, remote_producer=previous is not None) for append, boundaries in zip(appends, bound): for returned, boundary in zip(append.boundaries, boundaries): returned.plan = boundary.plan returned.declaration = boundary.declaration -def _bind_pending(chain: _Chain, *, consumer, terminal): +def _check_declared_gathers(appends, bound, *, remote_producer): + """A stage that declares its gather over attention TP must have one: at + its own entry, or at its producer's exit. A producer on another pipeline + rank runs it there.""" + line = [ + (append.origin, boundary) + for append, boundaries in zip(appends, bound) + if append.prepared_from is None + for boundary in boundaries + ] + for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]): + if consumer.declaration.attn_tp_gather is None or any( + _gathers_input(path) for path in consumer.plan.paths.values() + ): + continue + if producer is None and remote_producer: + continue + if producer is not None and any( + _gathers_output(path) for path in producer.plan.paths.values() + ): + continue + error = ValueError( + "an attention-TP gather is declared, but neither this stage's entry " + "nor its producer's exit gathers over attention TP" + ) + _note_origin(error, origin) + raise error + + +def _gathers_input(path): + return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input + + +def _gathers_output(path): + return getattr(path.output_move, "func", None) is update_attn_tp_gather_output + + +def _ended(declaration, final_read): + """The stack's last stage as the final read takes it: an FFN on this + rank's attention-TP slice leaves its output there for a read that reads + the slice and gathers what it read.""" + if declaration.kind is StageKind.FFN and getattr( + final_read, "reads_attn_tp_slices", False + ): + return replace(declaration, exit_rows=ExitRows.SLICE) + return declaration + + +def _bind_pending(chain: _Chain, *, consumer, terminal, final_read=None): pending = chain.pending if pending is None: return chain.pending = None try: declaration = replace(pending.declaration, terminal=terminal) + if terminal: + declaration = _ended(declaration, final_read) if pending.predecessor is None: incoming = _incoming(declaration) else: @@ -882,7 +952,12 @@ def _bind_pending(chain: _Chain, *, consumer, terminal): ) outgoing = _connect(declaration, consumer, residual_from=incoming) bound = _bind_stage( - declaration, pending.norm, incoming, outgoing, **pending.options + declaration, + pending.norm, + incoming, + outgoing, + final_read=final_read, + **pending.options, ) except Exception as error: _note_origin(error, pending.origin) diff --git a/python/sglang/srt/layers/layer_boundary/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py index de3ac0db81..58a6bc9599 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/residual/__init__.py @@ -70,6 +70,10 @@ class ResidualReadout(Protocol): is_plain_norm: Read is normalization/optional quantization without changing the residual, allowing compatible fused add+norm implementations. reads_before_dp_gather: Preserve this read on source rows before a DP gather. + reads_after_attn_tp_gather: Optional. The read needs every row of the + attention's, so it runs after any gather over attention TP: a + producer that would leave its output on this rank's attention-TP + slice returns it to the attention's rows instead. completing_fusions: Optional. ReadoutFusion kernels that complete the sum the input owes with the residual add ahead of this read, tried before the boundary's own all-reduce or reduce-scatter. diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py index e3f6fa49c7..3aefe947b2 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -96,6 +96,9 @@ class AttnBankState: # A read on this rank's attention-TP slice first tries the bank's kernels # that also run the collective around it. fuses_slice_collectives: bool = False + # The bank stays on each rank's attention-TP slice across layers; + # otherwise every rank writes every row and the attention reads them all. + reads_slices: bool = False def _aggregate( self, contribution, residual, norm, *, score_proj, score_norm, write @@ -216,6 +219,10 @@ class _AttnReadout(_BankReadout): """The attention input: the bank aggregation and this layer's input norm. A write layer snapshots the residual this read forms.""" + @property + def reads_after_attn_tp_gather(self): + return not self.state.reads_slices + @property def gathering_reads(self): state = self.state @@ -269,20 +276,29 @@ class AttnBankOutputRead: side's scoring parameters, then the final norm. A callable final norm for `residual_batch.final_norm`. - A stack that ends on this rank's attention-TP slice of the rows is read - there, against the bank rows this rank wrote, and what it read is - gathered: by the bank's kernel that does both when it takes the batch, - else by ``attn_tp_gather`` when that does, else over the attention-TP - group.""" + With ``reads_attn_tp_slices``, the stack may end on this rank's + attention-TP slice of the rows: such a batch is read there, against the + bank rows this rank wrote, and what it read is gathered: by the bank's + kernel that does both when it takes the batch, else by ``attn_tp_gather`` + when that does, else over the attention-TP group. Otherwise the stack's + last FFN brings its output to the attention's rows, in ``attn_tp_gather`` + when that takes the batch.""" def __init__( - self, bank: AttnBank, score_proj, score_norm, norm, attn_tp_gather=None + self, + bank: AttnBank, + score_proj, + score_norm, + norm, + attn_tp_gather=None, + reads_attn_tp_slices=False, ): self.bank = bank self.score_proj = score_proj self.score_norm = score_norm self.norm = norm self.attn_tp_gather = attn_tp_gather + self.reads_attn_tp_slices = reads_attn_tp_slices def __call__(self, hidden_states, residual=None): bank = self.bank.require() diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index 5b40bfaae1..6b143cf66e 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -44,6 +44,8 @@ class GatedResidualState: ffn_mix: The FFN stage's read. attn_combine: Writes the attention output back into the streams. ffn_combine: Writes the FFN output back into the streams. + attn_reads_every_row: Whether attn_mix needs every row of the + attention's, as a contribution computed for all of them does. """ expand: Callable @@ -51,6 +53,7 @@ class GatedResidualState: ffn_mix: Callable attn_combine: Callable ffn_combine: Callable + attn_reads_every_row: bool = False # Produced by a stage's read and consumed by that same stage's write-back. normed: Optional[torch.Tensor] = None @@ -110,6 +113,10 @@ class _AttnReadout: def __init__(self, state: GatedResidualState): self.state = state + @property + def reads_after_attn_tp_gather(self): + return self.state.attn_reads_every_row + def init_residual(self, hidden_states): return self.state.expand(hidden_states) diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 3a4378d845..696ac59978 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -33,7 +33,7 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import zero_copy_context from sglang.srt.layers.activation import SiluAndMul, SituAndMul -from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw +from sglang.srt.layers.attn_residual import aggregate_stream, get_cw from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, AuxHiddenStateList, @@ -42,15 +42,10 @@ from sglang.srt.layers.aux_hidden_states import ( from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata from sglang.srt.layers.dp_attention import ( - dp_gather_replicate, - dp_scatter, - get_global_dp_buffer, - get_local_dp_buffer, is_allocation_symmetric, is_dp_attention_enabled, ) from sglang.srt.layers.layer_boundary import ( - ExitRows, ReadoutFusion, SumGroup, append_stages, @@ -278,42 +273,6 @@ def _merge_weights_as_views( return merged, sizes -# K3 manages its own boundaries: the attn-res aggregation kernels replace -# input_layernorm / post_attention_layernorm, and the MLP/MoE modules -# gather/scatter around their own body: -# attention and the attn-res buffers stay in local (per-DP-rank) token space, -# the MLP/MoE runs on the DP-gathered global batch, and the delayed prefix_sum -# add stays local, applied after the scatter back. -def _dp_local_buffer_group(): - """Symmetric-memory group holding this DP rank's local token buffer.""" - parallel = get_parallel() - if parallel.tp_size == parallel.attn_dp_size: - return get_parallel().tp_group - return parallel.attn_tp_group - - -def _sp_all_gather_rows(hidden_states: torch.Tensor) -> torch.Tensor: - """Reassemble contiguous token shards, using the tuned K3 AG when covered.""" - group = get_parallel().attn_tp_group - hidden_states = hidden_states.contiguous() - full = k3_sp_collective.all_gather(hidden_states) - if full is None: - full = torch.empty( - (hidden_states.shape[0] * group.world_size, hidden_states.shape[1]), - dtype=hidden_states.dtype, - device=hidden_states.device, - ) - group.all_gather_into_tensor(full, hidden_states) - return full - - -def _sp_local_rows(hidden_states: torch.Tensor) -> slice: - """Full-batch row interval owned by this rank's contiguous token shard.""" - group = get_parallel().attn_tp_group - lo = group.rank_in_group * hidden_states.shape[0] - return slice(lo, lo + hidden_states.shape[0]) - - class KimiK3MLP(nn.Module): """K3 MLP; SiLU or SiTU activation.""" @@ -323,7 +282,6 @@ class KimiK3MLP(nn.Module): intermediate_size: int, hidden_act: str, quant_config: Optional[QuantizationConfig] = None, - reduce_results: bool = True, prefix: str = "", activation_situ_beta: float | None = None, activation_situ_linear_beta: float | None = None, @@ -355,7 +313,7 @@ class KimiK3MLP(nn.Module): hidden_size, bias=False, quant_config=quant_config, - reduce_results=reduce_results, + reduce_results=False, use_dp_attention_reduce=self._dense_attn_tp, prefix=f"{prefix}.down_proj", parallel_group=parallel_group, @@ -369,35 +327,11 @@ class KimiK3MLP(nn.Module): ) else: raise ValueError(f"Unsupported activation: {hidden_act}") - self._dp_attention = is_dp_attention_enabled() - def forward( - self, - hidden_states: torch.Tensor, - *, - prefix_sum: Optional[torch.Tensor] = None, - forward_batch: Optional[ForwardBatch] = None, - ) -> torch.Tensor: - # DP attention only when driven from the decoder layer (forward_batch - # given); the shared-experts instance inside KimiK3MoE passes None and - # runs on the already-gathered buffer. - use_dp = ( - self._dp_attention and forward_batch is not None and not self._dense_attn_tp - ) - if use_dp: - local_hidden_states = hidden_states - hidden_states = get_global_dp_buffer(get_parallel().tp_group) - dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: gate_up, _ = self.gate_up_proj(hidden_states) hidden_states = self.act_fn(gate_up) hidden_states, _ = self.down_proj(hidden_states) - if use_dp: - global_out = hidden_states - hidden_states = get_local_dp_buffer(_dp_local_buffer_group()) - dp_scatter(hidden_states, global_out, forward_batch) - # TODO(dark): maybe fuse residual with all reduce of down projection - if prefix_sum is not None: - hidden_states = hidden_states + prefix_sum return hidden_states @@ -447,9 +381,10 @@ def _k3_reduce_scatter_add(hidden_states, residual, forward_batch): def _route_sp_o_proj_output(o_proj: RowParallelLinear) -> None: - """o_proj under SP-MoE.""" - # o_proj emits TP-partial sums, which the stage boundary completes. - o_proj.reduce_results = False + """Under SP-MoE, write o_proj's TP-partial output into K3's persistent + symmetric buffer whenever K3's tuned reduce-scatter for that row count + reads through it (the bank's FFN read); otherwise the stage boundary + reduce-scatters it from regular memory.""" if k3_sp_collective.enabled(): # The table selects NVLS pull RS for larger token buckets. # Only those o_proj outputs come from the persistent symmetric @@ -492,13 +427,8 @@ class KimiK3MoE(nn.Module): prefix: str = "", layer_idx: int = 0, alt_stream: Optional[torch.cuda.Stream] = None, - reduce_results: bool = True, ): super().__init__() - # Whether this block completes the TP sum of a non-latent output; a - # stage boundary completes it otherwise. A latent block completes its - # sums together with the latent reduction either way. - self.reduce_results = reduce_results hidden_size = config.hidden_size moe_intermediate_size = config.moe_intermediate_size moe_renormalize = config.moe_renormalize @@ -507,7 +437,6 @@ class KimiK3MoE(nn.Module): self.num_shared_experts = config.num_shared_experts self.layer_idx = layer_idx self.alt_stream = alt_stream - self._dp_attention = is_dp_attention_enabled() self.use_latent_moe = config.routed_expert_hidden_size is not None # Merged front weight ([H, gate_up + E + latent]), built after weight @@ -614,8 +543,8 @@ class KimiK3MoE(nn.Module): # EP a2a backends move each row to its experts directly, so the MoE # region consumes whatever rows this rank holds (SP-MoE shard or - # DP-local batch) with every global token dispatched exactly once. - # No DP gather and no TP reduce anywhere in the region. + # DP-local batch). No DP gather and no TP reduce anywhere in the + # region. _a2a_backend = get_moe_a2a_backend() self._ep_a2a = ( _a2a_backend.is_megamoe() @@ -679,7 +608,6 @@ class KimiK3MoE(nn.Module): intermediate_size=shared_intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, - reduce_results=False, prefix=f"{prefix}.shared_experts", activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, @@ -859,11 +787,8 @@ class KimiK3MoE(nn.Module): ) from sglang.srt.runtime_context import get_parallel - # In SP-MoE mode (KimiK3DecoderLayer reduce-scatters the o_proj - # output) the incoming rows are already this rank's token shard, so - # the fused a2a below dispatches each token exactly once. On the - # non-scattered fallback path the rows are the full batch (redundant - # across ranks but correct). + # Under SP-MoE these rows are this rank's shard of the batch; an unpadded + # batch stays on every row. num_tokens = routed_input.shape[0] num_max_tokens_per_rank = ( envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get() @@ -1272,12 +1197,6 @@ class KimiK3MoE(nn.Module): wait_and_finalize_shared_experts() if shared_output is not None: expert_output = expert_output + shared_output - # EP combine and the shared-expert subgroup have already completed - # each source token. A global TP reduction would mix token shards. - if self.tp_size > 1 and not self._ep_a2a and self.reduce_results: - expert_output = tensor_model_parallel_all_reduce(expert_output) - if prefix_sum is not None: - expert_output = expert_output + prefix_sum return expert_output # Latent MoE: compress after routing, before experts @@ -1541,40 +1460,17 @@ class KimiK3MoE(nn.Module): hidden_states: torch.Tensor, *, prefix_sum: Optional[torch.Tensor] = None, - forward_batch: Optional[ForwardBatch] = None, ) -> torch.Tensor: - """A pending prefix_sum is always consumed here: folded into the - 3-way JIT tail add when covered, plain adds otherwise (bit-identical - either way). - - Under DP attention with TP-sharded experts (a2a none, forward_batch - given) the experts run on the DP-gathered global batch — the internal - reduces stay over the full TP group, which is exactly right in - gathered space — while prefix_sum stays in local token space, added - after the scatter back. EP a2a backends skip the gather entirely: - dispatching the DP-local rows (or the SP-MoE shard of them the - decoder layer already produced) covers every global token exactly - once, and prefix_sum is consumed in the tail add like the non-DP - path — gathering first would just replicate the whole batch onto - every rank (tp-fold redundant compute + a2a traffic).""" + """A latent MoE, which writes the next stream itself, takes the pending + prefix_sum: folded into the 3-way JIT tail add when covered, plain adds + otherwise (bit-identical either way). A non-latent MoE leaves the + residual add to its stage boundary and takes none.""" num_tokens, hidden_size = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_size) - use_dp = self._dp_attention and forward_batch is not None and not self._ep_a2a - if use_dp: - local_hidden_states = hidden_states - hidden_states = get_global_dp_buffer(get_parallel().tp_group) - dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) - dp_prefix_sum, prefix_sum = prefix_sum, None if hidden_states.shape[0] > 0 and self._eligible_for_fused_front: out = self._forward_fused(hidden_states, prefix_sum=prefix_sum) else: out = self._forward_unfused(hidden_states, prefix_sum=prefix_sum) - if use_dp: - global_out = out - out = get_local_dp_buffer(_dp_local_buffer_group()) - dp_scatter(out, global_out, forward_batch) - if dp_prefix_sum is not None: - out = out + dp_prefix_sum return out.view(num_tokens, hidden_size) @@ -1591,7 +1487,6 @@ class KimiK3DeltaAttention(nn.Module): prefix: str = "", all_reduce_fusion: bool = False, bfa_alt_stream: Optional[torch.cuda.Stream] = None, - reduce_results: bool = True, sp_moe: bool = False, **kwargs, ) -> None: @@ -1844,20 +1739,12 @@ class KimiK3DeltaAttention(nn.Module): projection_size, self.hidden_size, bias=False, - # SGLANG_K3_AR_FUSION: keep the o_proj output TP-partial and - # complete the reduce at the decoder layer via the fused MNNVL - # all-reduce (which can fold the attn-res prefix add in). Only - # valid when the attn TP group is the full TP group (the fused - # comm lives there). - reduce_results=reduce_results and not self.all_reduce_fusion, + # The stage boundary completes the attention-TP sum. + reduce_results=False, quant_config=quant_config, parallel_group="attn_tp", - # Reduce within the attn-TP group: the default full-TP collective - # is the wrong group at attn_tp>1 (sums across DP groups) and - # deadlocks idle DP ranks. Off under all_reduce_fusion: the fused - # AR reduces in place out of a symmetric buffer, and this flag - # would wrap the GEMM in its own symm_ctx, defeating the - # caller-owned buffer. + # Symmetric memory of the attention-TP group, which sums the output; + # off under all_reduce_fusion, whose caller owns the output buffer. use_dp_attention_reduce=not self.all_reduce_fusion, prefix=f"{prefix}.o_proj", ) @@ -1865,7 +1752,6 @@ class KimiK3DeltaAttention(nn.Module): # the fused AR reduces o_proj's output in place out of a symmetric # buffer, which needs the GEMM to write into caller-owned storage self.all_reduce_fusion = False - self.o_proj.reduce_results = reduce_results self.o_proj.use_dp_attention_reduce = True conv_weights = self.qkv_conv1d.weight.squeeze(1) bias = self.qkv_conv1d.bias @@ -2248,7 +2134,6 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, gate_alt_stream: Optional[torch.cuda.Stream] = None, - reduce_results: bool = True, sp_moe: bool = False, ) -> None: # ModelSlim can quantize K3 latent projections while still storing @@ -2272,7 +2157,8 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): q_lora_rank=config.q_lora_rank, kv_lora_rank=config.kv_lora_rank, skip_rope=True, - reduce_results=reduce_results and not self.all_reduce_fusion, + # The stage boundary completes the attention-TP sum. + reduce_results=False, alt_stream=alt_stream, ) if split_gguf_kv_b: @@ -2326,8 +2212,6 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): # the fused AR reduces o_proj's output in place out of a symmetric # buffer, which needs the GEMM to write into caller-owned storage self.all_reduce_fusion = False - self.o_proj.reduce_results = reduce_results - self.o_proj.use_dp_attention_reduce = True if self.all_reduce_fusion: # Hand the GEMM a slice of the persistent symmetric buffer # (k3_ar_fusion.symm_buffer); the fused AR reduces it in place. @@ -2347,9 +2231,8 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): self.o_proj.forward = _symm_o_proj_forward else: - # K3 owns its output reduction, so o_proj must reduce within the - # attn-TP group itself: the default full-TP collective is the wrong - # group at attn_tp>1 and deadlocks idle DP ranks. + # Allocate o_proj's output in the attention-TP group's symmetric + # memory, where the stage boundary sums it. self.o_proj.use_dp_attention_reduce = True if self.use_output_gate: projection_size = config.num_attention_heads * config.v_head_dim @@ -2521,12 +2404,6 @@ def _carries_bank_slices(config: KimiLinearConfig) -> bool: return not (spec.is_eagle3() or spec.is_dflash_family()) -def _uses_stage_boundaries(config: KimiLinearConfig) -> bool: - """Whether the layers build stage boundaries, which every configuration - does.""" - return True - - class KimiK3DecoderLayer(nn.Module): """Decoder layer carrying the K3 attention-residual stream.""" @@ -2544,19 +2421,10 @@ class KimiK3DecoderLayer(nn.Module): self.layer_idx = layer_idx self._dp_attention = is_dp_attention_enabled() self._is_moe_layer = _is_moe_layer(config, layer_idx) - self._stage_boundaries = _uses_stage_boundaries(config) - # SP-MoE (EP a2a backend): o_proj defers its attention-TP reduction; - # this layer completes it as a reduce-scatter so the whole MoE region - # runs on 1/attn_tp of the rows, then all-gathers back after the tail - # add. RS+AG moves the same bytes the AR did, the shared-expert AR - # disappears via tp1 weights, and each rank dispatches only its shard - # through the a2a. Same under DP attention. Dense layers excluded: no - # per-token decomposition survives a token shard. + # SP-MoE: with an EP a2a backend and attention TP, a MoE layer runs on + # this rank's attention-TP shard of the rows. self._sp_moe = self._is_moe_layer and _shards_moe_rows() - # Mutually exclusive with SP-MoE: both complete o_proj's deferred - # reduction, but SP-MoE reduce-scatters to a shard whereas the fusion - # produces the full batch in a symm buffer. self.all_reduce_fusion = not self._sp_moe and _fuses_attn_all_reduce(config) # Attention @@ -2568,7 +2436,6 @@ class KimiK3DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, - reduce_results=not self._stage_boundaries, sp_moe=self._sp_moe, # Shared with the MLA gate stream: KDA and MLA layers never # run concurrently within one forward, so the stream is free. @@ -2581,14 +2448,13 @@ class KimiK3DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, - reduce_results=not self._stage_boundaries, sp_moe=self._sp_moe, alt_stream=alt_streams[1] if alt_streams is not None else None, gate_alt_stream=alt_streams[2] if alt_streams is not None else None, ) # the attention drops the fusion when its o_proj cannot write into - # caller-owned storage; the layer's own AR call-site must agree + # caller-owned storage; the FFN read's fused all-reduce must agree self.all_reduce_fusion = self.self_attn.all_reduce_fusion # MLP / MoE @@ -2599,7 +2465,6 @@ class KimiK3DecoderLayer(nn.Module): layer_idx=layer_idx, prefix=f"{prefix}.mlp", alt_stream=alt_streams[0] if alt_streams is not None else None, - reduce_results=not self._stage_boundaries, ) else: self.mlp = KimiK3MLP( @@ -2607,7 +2472,6 @@ class KimiK3DecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, - reduce_results=not self._stage_boundaries, prefix=f"{prefix}.mlp", activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, @@ -2646,25 +2510,22 @@ class KimiK3DecoderLayer(nn.Module): # writes the next stream itself; one gathered over attention DP returns # its output to this rank's tokens first. self._ffn_writes_stream = ( - self._stage_boundaries - and self.use_attn_residuals + self.use_attn_residuals and self._is_moe_layer and self.mlp.use_latent_moe and not (self._dp_attention and not self.mlp._ep_a2a) ) - if self._stage_boundaries: - self._declare_stages(config, layer_idx, attn_bank) + self._declare_stages(config, layer_idx, attn_bank) def _declare_stages(self, config, layer_idx, attn_bank): """Declare this layer's attention and FFN stages: the reads and updates - around them, the rows the FFN takes and leaves, and the kernels K3 - supplies for the bank path.""" + around them, the rows the FFN takes, and the kernels K3 supplies for + the bank path.""" # Under attention DP the FFN input is read on this rank's rows # once the attention's sum is complete, then gathered. attn_ops = {} ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) carries = _carries_bank_slices(config) - bank_sp_moe = self._sp_moe and self.use_attn_residuals tuned_gather = ( k3_sp_collective.all_gather if k3_sp_collective.enabled() else None ) @@ -2680,15 +2541,18 @@ class KimiK3DecoderLayer(nn.Module): fuses_slice_collectives=self._sp_moe and k3_sp_collective.enabled() and envs.SGLANG_K3_SP_ATTN_RES.get(), + reads_slices=carries, ).residual_ops() attn_ops = dict( read=bank_ops.attn_readout, update=bank_ops.attn_update, - # An input read on the shard an SP-MoE layer left is gathered - # in K3's tuned all-gather when it takes the batch. + # The rows an SP-MoE layer left on each rank's shard are + # gathered in K3's tuned all-gather when it takes the batch: + # before this read, or after it while the bank stays on the + # shard. attn_tp_gather=( tuned_gather - if carries and _is_moe_layer(config, layer_idx - 1) + if _shards_moe_rows() and _is_moe_layer(config, layer_idx - 1) else None ), ) @@ -2711,23 +2575,9 @@ class KimiK3DecoderLayer(nn.Module): if self._is_moe_layer else get_group_rank_size(self.mlp.down_proj.tp_group)[1] ), - # SP-MoE runs on this rank's attention-TP shard of the - # rows; on the bank path, whose reads write the bank on - # every row, its output returns to all of them, unless - # the bank stays on the shard, read there to the end. - exit_rows=( - (ExitRows.SLICE if carries else ExitRows.ATTENTION) - if bank_sp_moe - else None - ), # A latent MoE completes its output sum together with # the latent reduction its norm needs. output_complete=self._is_moe_layer and self.mlp.use_latent_moe, - # The bank path's gather back to every row, in K3's - # tuned all-gather when it takes the batch. - attn_tp_gather=( - tuned_gather if bank_sp_moe and not carries else None - ), ), self.post_attention_layernorm, ), @@ -2746,39 +2596,6 @@ class KimiK3DecoderLayer(nn.Module): ) return () - def _finish_attn_reduce( - self, - attn_out: torch.Tensor, - allow_scatter: bool, - residual: Optional[torch.Tensor] = None, - ) -> tuple[torch.Tensor, int, bool]: - """Complete o_proj's deferred TP reduction under SP-MoE. - - Returns (reduced tensor, shard row offset, residual_fused); offset is - -1 when the result covers the full batch (non-SP mode, or fallback - all-reduce for row counts not divisible by attn_tp).""" - if not self._sp_moe: - return attn_out, -1, False - group = get_parallel().attn_tp_group - num_tokens = attn_out.shape[0] - if allow_scatter and num_tokens > 0 and num_tokens % group.world_size == 0: - shard = num_tokens // group.world_size - custom_out = k3_sp_collective.reduce_scatter_res(attn_out, residual) - if custom_out is not None: - return ( - custom_out, - group.rank_in_group * shard, - residual is not None, - ) - out = torch.empty( - (shard, attn_out.shape[1]), - dtype=attn_out.dtype, - device=attn_out.device, - ) - group.reduce_scatter_tensor(out, attn_out) - return out, group.rank_in_group * shard, False - return group.all_reduce(attn_out), -1, False - def _run_self_attn( self, hidden_states: torch.Tensor, @@ -2821,201 +2638,25 @@ class KimiK3DecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - attn_res: Optional[AttnResidual], zero_allocator: BumpAllocator, - input_sharded: bool = False, - keep_sharded: bool = False, capture_gathered: Optional[AuxHiddenStateAccumulator] = None, - ) -> tuple[torch.Tensor, Optional[torch.Tensor], bool]: - if attn_res is not None: - return self._forward_attn_residual( - positions, - hidden_states, - residual, - attn_res, - forward_batch, - zero_allocator, - input_sharded, - keep_sharded, - ) - - assert not input_sharded - if self._stage_boundaries: - hidden_states = self.attn_boundary.prepare( - hidden_states, forward_batch, capture_gathered=capture_gathered - ) - hidden_states = self._run_self_attn( - hidden_states, positions, forward_batch, zero_allocator - ) - hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) - hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + ) -> torch.Tensor: + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture_gathered=capture_gathered + ) + hidden_states = self._run_self_attn( + hidden_states, positions, forward_batch, zero_allocator + ) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + if self._ffn_writes_stream: hidden_states = self.mlp( hidden_states, - prefix_sum=( - residual_batch.written_residual(forward_batch) - if self._ffn_writes_stream - else None - ), + prefix_sum=residual_batch.written_residual(forward_batch), ) - return self.ffn_boundary.finish(hidden_states, forward_batch), None, False - - # Standard residual path - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - - hidden_states = self._run_self_attn( - hidden_states, positions, forward_batch, zero_allocator - ) - # standard path returns a full-size residual to the next layer, so - # complete the deferred o_proj reduction as a plain all-reduce - hidden_states, _, _ = self._finish_attn_reduce( - hidden_states, allow_scatter=False - ) - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - return hidden_states, residual, False - - def _forward_attn_residual( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - prefix_sum: Optional[torch.Tensor], - attn_res: AttnResidual, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - input_sharded: bool, - keep_sharded: bool, - ) -> tuple[torch.Tensor, Optional[torch.Tensor], bool]: - # Between attn-res layers hidden_states carries the previous layer's - # un-added MLP delta and prefix_sum the prefix it extends (None at - # stream start / PP entry, where hidden_states already is the head). - - # ---- Aggregation 1: attention side. Write layers snapshot the - # pre-attention prefix into the bank in the same call (fused into - # the fast kernel; standalone copy on other paths). ---- - if input_sharded: - assert self._sp_moe - input_rows = _sp_local_rows(hidden_states) - fused_ag = attn_res.forward_sp_all_gather( - hidden_states, - prefix_sum, - self.self_attention_res_proj, - self.self_attention_res_norm, - self.input_layernorm, - rows=input_rows, - write=self.is_block_write_layer, - ) - if fused_ag is not None: - hidden_states, prefix_sum = fused_ag - else: - hidden_states, prefix_sum = attn_res.forward( - hidden_states, - prefix_sum, - self.self_attention_res_proj, - self.self_attention_res_norm, - self.input_layernorm, - rows=input_rows, - write=self.is_block_write_layer, - ) - # Aggregate/norm and snapshot only this rank's rows, then - # gather the normalized tensor consumed by attention. - hidden_states = _sp_all_gather_rows(hidden_states) - else: - hidden_states, prefix_sum = attn_res.forward( - hidden_states, - prefix_sum, - self.self_attention_res_proj, - self.self_attention_res_norm, - self.input_layernorm, - write=self.is_block_write_layer, - ) - if self.is_block_write_layer: - prefix_sum = None - - # ---- Attention ---- - hidden_states = self._run_self_attn( - hidden_states, positions, forward_batch, zero_allocator - ) - - # ---- Complete o_proj's deferred reduction ---- - # SP-MoE takes precedence (reduce-scatter to this rank's token shard); - # otherwise the fused all-reduce when enabled; otherwise o_proj already - # reduced itself (use_dp_attention_reduce, on when neither is active). - rows = None - shard_lo = -1 - agg2_fused = False - if self._sp_moe: - group = get_parallel().attn_tp_group - if ( - hidden_states.shape[0] > 0 - and hidden_states.shape[0] % group.world_size == 0 - ): - shard = hidden_states.shape[0] // group.world_size - fused_rows = slice( - group.rank_in_group * shard, - (group.rank_in_group + 1) * shard, - ) - fused_rs = attn_res.forward_sp_reduce_scatter( - hidden_states, - prefix_sum, - self.mlp_res_proj, - self.mlp_res_norm, - self.post_attention_layernorm, - rows=fused_rows, - ) - else: - fused_rs = None - if fused_rs is not None: - hidden_states, prefix_sum = fused_rs - rows = fused_rows - shard_lo = fused_rows.start - agg2_fused = True - else: - hidden_states, shard_lo, residual_fused = self._finish_attn_reduce( - hidden_states, allow_scatter=True, residual=prefix_sum - ) - if shard_lo >= 0: - rows = slice(shard_lo, shard_lo + hidden_states.shape[0]) - if residual_fused: - prefix_sum = None - elif prefix_sum is not None: - # Shard carry already holds the destination-local prefix; - # the first sharded layer still holds a full-batch prefix. - if prefix_sum.shape[0] != hidden_states.shape[0]: - prefix_sum = prefix_sum[rows] - elif self.all_reduce_fusion: - # Complete the o_proj reduce here, folding the pending prefix add - # into the fused all-reduce; attn_res then takes the pre-added - # tensor through its prefix_sum=None branch (same semantics: - # (normed, new_prefix) with new_prefix = prefix + attn_out). - hidden_states = k3_ar_fusion.all_reduce(hidden_states, prefix_sum) - prefix_sum = None - - # ---- Aggregation 2: MLP side (on the shard under SP-MoE) ---- - if not agg2_fused: - hidden_states, prefix_sum = attn_res.forward( - hidden_states, - prefix_sum, - self.mlp_res_proj, - self.mlp_res_norm, - self.post_attention_layernorm, - rows=rows, - ) - - # ---- MLP (consumes +prefix_sum: MoE folds it into the 3-way tail - # add, dense adds it after down_proj) ---- - out = self.mlp( - hidden_states, prefix_sum=prefix_sum, forward_batch=forward_batch - ) - if shard_lo >= 0: - if keep_sharded: - return out, None, True - out = _sp_all_gather_rows(out) - return out, None, False + hidden_states = self.mlp(hidden_states) + return self.ffn_boundary.finish(hidden_states, forward_batch) class KimiK3LinearModel(nn.Module): @@ -3032,7 +2673,6 @@ class KimiK3LinearModel(nn.Module): self.pp_group = get_parallel().pp_group self.dspark_layers_to_capture: Optional[list[int]] = None self._dp_attention = is_dp_attention_enabled() - self._stage_boundaries = _uses_stage_boundaries(config) self.carries_bank_slices = _carries_bank_slices(config) if self.pp_group.is_first_rank: @@ -3062,24 +2702,9 @@ class KimiK3LinearModel(nn.Module): # The attention-residual bank of one forward, which every layer's # stage boundaries read. - self.attn_bank = ( - AttnBank() - if self._stage_boundaries and config.attn_res_block_size is not None - else None - ) - self.layers, self.start_layer, self.end_layer = make_pp_layers( - config.num_hidden_layers, - lambda idx, prefix: KimiK3DecoderLayer( - layer_idx=idx, - config=config, - quant_config=quant_config, - prefix=prefix, - alt_streams=self.alt_streams, - attn_bank=self.attn_bank, - ), - prefix=f"{prefix}.layers", - ) - + self.attn_bank = AttnBank() if config.attn_res_block_size is not None else None + # The layer stack's last FFN leaves its output on the rows the final + # read takes. if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) if config.attn_res_block_size is not None: @@ -3101,15 +2726,34 @@ class KimiK3LinearModel(nn.Module): self.output_attn_res_proj, self.output_attn_res_norm, self.norm, + # The rows a last SP-MoE layer leaves on each rank's + # shard are gathered in K3's tuned all-gather when it + # takes the batch. attn_tp_gather=( k3_sp_collective.all_gather - if self.carries_bank_slices + if k3_sp_collective.enabled() + and _shards_moe_rows() + and _is_moe_layer(config, config.num_hidden_layers - 1) else None ), + reads_attn_tp_slices=self.carries_bank_slices, ) ) else: self.norm = PPMissingLayer() + self.layers, self.start_layer, self.end_layer = make_pp_layers( + config.num_hidden_layers, + lambda idx, prefix: KimiK3DecoderLayer( + layer_idx=idx, + config=config, + quant_config=quant_config, + prefix=prefix, + alt_streams=self.alt_streams, + attn_bank=self.attn_bank, + ), + prefix=f"{prefix}.layers", + final_read=self._final_read if self.pp_group.is_last_rank else None, + ) def forward( self, @@ -3119,32 +2763,22 @@ class KimiK3LinearModel(nn.Module): inputs_embeds: torch.Tensor | None = None, pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> torch.Tensor: - residual = None if get_parallel().pp_group.is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - if self._stage_boundaries: - residual_batch.start(forward_batch) + residual_batch.start(forward_batch) elif self.attn_bank is not None: # The bank's wire carries the stream head written: the sending rank # folds it in before the handoff. hidden_states = residual_batch.set_written( pp_proxy_tensors["hidden_states"], forward_batch ) - elif self._stage_boundaries: + else: hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( pp_proxy_tensors, forward_batch ) - else: - assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - if self.config.attn_res_block_size is None: - residual = pp_proxy_tensors["residual"] - if TYPE_CHECKING: - assert isinstance(hidden_states, torch.Tensor) - assert isinstance(residual, torch.Tensor | None) total_num_layers = self.end_layer - self.start_layer device = hidden_states.device @@ -3154,8 +2788,7 @@ class KimiK3LinearModel(nn.Module): device=device, ) - attn_res = None - if self.config.attn_res_block_size is not None: + if self.attn_bank is not None: attn_res_block_num = _cdiv(self.end_layer, self.config.attn_res_block_size) # A later pipeline rank inherits the rows the earlier ones banked. inherited = ( @@ -3163,24 +2796,8 @@ class KimiK3LinearModel(nn.Module): if self.pp_group.is_first_rank else pp_proxy_tensors["attn_res_bank"] ) - if self.attn_bank is not None: - self.attn_bank.open(hidden_states, attn_res_block_num, inherited) - else: - attn_res = AttnResidual( - hidden_states, attn_res_block_num, block_residual=inherited - ) + self.attn_bank.open(hidden_states, attn_res_block_num, inherited) - # Carry the raw residual stream as a token shard across consecutive - # SP-MoE layers. PP transfer and dspark capture require full tensors, - # so those uncommon paths keep the established gather-per-layer flow. - sp_attn_res = ( - attn_res is not None - and envs.SGLANG_K3_SP_ATTN_RES.get() - and self.pp_group.world_size == 1 - and self.dspark_layers_to_capture is None - and k3_sp_collective.enabled() - ) - sp_sharded = False packs_aux = self.packs_aux_hidden_states aux_hidden_states: AuxHiddenStateAccumulator = ( AuxHiddenStatePacker.for_batch( @@ -3194,7 +2811,7 @@ class KimiK3LinearModel(nn.Module): # next layer's attention input reads, captured there on every row (the # final norm's, after the last layer). The bank's is the mixture the # next read would form, captured after the layer. - captures_at_input = self._stage_boundaries and self.attn_bank is None + captures_at_input = self.attn_bank is None if ( self.dspark_layers_to_capture is not None and not self.pp_group.is_first_rank @@ -3204,27 +2821,16 @@ class KimiK3LinearModel(nn.Module): if self.start_layer - 1 in captures and not captures_at_input: aux_hidden_states.append( self._dspark_capture_stream( - self.start_layer - 1, - hidden_states, - residual, - attn_res, - forward_batch, + self.start_layer - 1, hidden_states, forward_batch ) ) for i in range(self.start_layer, self.end_layer): - if sp_sharded and not self.layers[i]._sp_moe: - hidden_states = _sp_all_gather_rows(hidden_states) - sp_sharded = False with get_global_expert_distribution_recorder().with_current_layer(i): - hidden_states, residual, sp_sharded = self.layers[i]( + hidden_states = self.layers[i]( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=residual, - attn_res=attn_res, zero_allocator=zero_allocator, - input_sharded=sp_sharded, - keep_sharded=sp_attn_res, capture_gathered=( aux_hidden_states if captures_at_input and i - 1 in captures @@ -3237,12 +2843,10 @@ class KimiK3LinearModel(nn.Module): and (i + 1 < self.end_layer or self.pp_group.is_last_rank) ): aux_hidden_states.append( - self._dspark_capture_stream( - i, hidden_states, residual, attn_res, forward_batch - ) + self._dspark_capture_stream(i, hidden_states, forward_batch) ) - if not self.pp_group.is_last_rank and self._stage_boundaries: + if not self.pp_group.is_last_rank: if self.attn_bank is not None: # The wire carries the stream head beside the bank. hidden_states = residual_batch.fold(hidden_states, forward_batch) @@ -3257,77 +2861,20 @@ class KimiK3LinearModel(nn.Module): aux_hidden_states, dim=-1 ) return proxy_tensors - if not self.pp_group.is_last_rank: - assert not sp_sharded - if attn_res is not None: - if residual is not None: - # Materialize the delayed MLP add: the wire carries the - # full stream head (bit-identical to the fused fold). - hidden_states = residual + hidden_states - proxy_tensors = { - "hidden_states": hidden_states, - "attn_res_bank": attn_res.block_residual, - } - else: - proxy_tensors = {"hidden_states": hidden_states, "residual": residual} - if aux_hidden_states: - proxy_tensors["dspark_hidden_states"] = torch.cat( - aux_hidden_states, dim=-1 - ) - return PPProxyTensors(proxy_tensors) - if self._stage_boundaries: - hidden_states = residual_batch.final_norm( - hidden_states, - forward_batch, - self._final_read, - capture=( - aux_hidden_states.capture - if captures_at_input and self.end_layer - 1 in captures - else None - ), - skip_empty=True, - ) - if self.attn_bank is not None: - self.attn_bank.close() - elif hidden_states.shape[0] != 0: - if attn_res is not None: - # ---- Final aggregation (output side, folds delayed add) ---- - if sp_sharded: - output_rows = _sp_local_rows(hidden_states) - fused_output = attn_res.forward_sp_all_gather( - hidden_states, - residual, - self.output_attn_res_proj, - self.output_attn_res_norm, - self.norm, - rows=output_rows, - ) - if fused_output is not None: - hidden_states, _ = fused_output - else: - hidden_states, _ = attn_res.forward( - hidden_states, - residual, - self.output_attn_res_proj, - self.output_attn_res_norm, - self.norm, - rows=output_rows, - ) - hidden_states = _sp_all_gather_rows(hidden_states) - else: - hidden_states, _ = attn_res.forward( - hidden_states, - residual, - self.output_attn_res_proj, - self.output_attn_res_norm, - self.norm, - ) - else: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = residual_batch.final_norm( + hidden_states, + forward_batch, + self._final_read, + capture=( + aux_hidden_states.capture + if captures_at_input and self.end_layer - 1 in captures + else None + ), + skip_empty=True, + ) + if self.attn_bank is not None: + self.attn_bank.close() if packs_aux: return hidden_states, aux_hidden_states.finalize() @@ -3346,22 +2893,13 @@ class KimiK3LinearModel(nn.Module): self, layer_idx: int, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - attn_res: Optional[AttnResidual], forward_batch: ForwardBatch, ) -> torch.Tensor: - """Stream value after `layer_idx`: the pre-norm mixture its next - consumer would compute (next layer's attention side; output side - for the last layer).""" - if self._stage_boundaries: - # The bank path; the standard one captures at the next input. - hidden_states = residual_batch.snapshot(hidden_states, forward_batch) - attn_res, residual = self.attn_bank.require(), None - if attn_res is None: - return hidden_states if residual is None else hidden_states + residual - if residual is not None: - # Materialize a delayed MLP add (mirrors the PP-wire fold). - hidden_states = residual + hidden_states + """Stream value after `layer_idx` on the bank path: the pre-norm + mixture its next consumer would compute (next layer's attention side; + output side for the last layer). The standard path captures at the + next input instead.""" + hidden_states = residual_batch.snapshot(hidden_states, forward_batch) if layer_idx + 1 < self.end_layer: next_layer = self.layers[layer_idx + 1] score_proj = next_layer.self_attention_res_proj @@ -3373,7 +2911,11 @@ class KimiK3LinearModel(nn.Module): score_norm = self.output_attn_res_norm nvb = _cdiv(self.end_layer, self.config.attn_res_block_size) return aggregate_stream( - hidden_states, attn_res.block_residual, nvb, score_proj, score_norm + hidden_states, + self.attn_bank.require().block_residual, + nvb, + score_proj, + score_norm, ) diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 24792dfc81..7b2069359a 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -37,7 +37,6 @@ from sglang.srt.layers.hyperconnection import ( HyperConnectionConfig, ) from sglang.srt.layers.layer_boundary import ( - ExitRows, GatedResidualState, append_stages, declare_attn, @@ -1434,13 +1433,7 @@ def _has_ple(layer_id, config) -> bool: return (layer_id + 1) in config.ple_layer_ids -def _ffn_exit_rows(layer_id, config): - """A PLE layer's read adds an embedding computed for every row, so the FFN - before it hands its output on as full attention rows.""" - return ExitRows.ATTENTION if _has_ple(layer_id + 1, config) else None - - -def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config): +def _build_qwen4_exp_stages(residual, *, sparse): """The attention and FFN stage boundaries of one gated hyper-connection layer. Each read normalizes the streams itself, so neither stage binds a norm, and @@ -1458,7 +1451,6 @@ def _build_qwen4_exp_stages(residual, *, sparse, layer_id, config): next_layer_sparse=sparse, read=residual.ffn_readout, update=residual.ffn_update, - exit_rows=_ffn_exit_rows(layer_id, config), ), None, ), @@ -1526,10 +1518,10 @@ class Qwen4ExpLayerExtensionMixin: ffn_mix=self.mlp_hyper_connection.mix, attn_combine=self.attn_hyper_connection.combine, ffn_combine=self.mlp_hyper_connection.combine, + # The PLE embedding is computed for every row. + attn_reads_every_row=self.ple is not None, ).residual_ops(), sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock), - layer_id=layer_id, - config=config, ) def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 0d22705f49..e4db549a8b 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1590,13 +1590,15 @@ def make_layers( prefix: str = "", return_tuple: bool = False, offloader_kwargs: Optional[Dict[str, Any]] = None, + final_read: Optional[Any] = None, ) -> Tuple[torch.nn.Module, int, int]: """Make a list of layers with the given layer function. The local layers are built inside one layer stack, so layers that declare stage boundaries connect in order without naming their neighbours. Across a pipeline stage boundary the stack learns the neighbouring stage from the - layer itself, built again on the meta device. + layer itself, built again on the meta device. ``final_read`` is the + stack's terminal read when it is not a plain final norm (see layer_stack). """ # circular imports from sglang.srt.distributed import get_pp_indices @@ -1623,6 +1625,7 @@ def make_layers( with layer_stack( previous_layers=[neighbour(idx) for idx in reversed(range(start_layer))], next_layers=[neighbour(idx) for idx in range(end_layer, num_hidden_layers)], + final_read=final_read, ): modules = torch.nn.ModuleList( [PPMissingLayer(return_tuple=return_tuple) for _ in range(start_layer)] @@ -1649,6 +1652,7 @@ def make_pp_layers( prefix: str = "", return_tuple: bool = False, offloader_kwargs: Optional[Dict[str, Any]] = None, + final_read: Optional[Any] = None, ) -> Tuple[torch.nn.Module, int, int]: """Make this pipeline stage's layers, and return them with the stage's range. @@ -1663,6 +1667,7 @@ def make_pp_layers( prefix=prefix, return_tuple=return_tuple, offloader_kwargs=offloader_kwargs, + final_read=final_read, ) diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index a95bb4daf3..4af738eabd 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -457,48 +457,73 @@ class TestUnpaddedBatches(CustomTestCase): self.assertNotIn(BatchVariant.UNPADDED, ffn.plan.paths) -class TestSliceExitRows(CustomTestCase): - """An FFN on this rank's attention-TP slice that declares ExitRows.SLICE - leaves its output there, the stack's last one too, where the final read - then takes it; the next layer's attention reads it there and gathers what - it read, in the declared gather when that takes the batch.""" +class TestRowsTheConsumerReads(CustomTestCase): + """An FFN on this rank's attention-TP slice leaves its output there unless + the stage after it needs the attention's rows: a read that needs every row + (reads_after_attn_tp_gather) has the FFN's exit gather them, in that + stage's declared gather. At the stack's end the model's final read takes + that place: one that reads the slice (reads_attn_tp_slices) takes the + last FFN's output there, and its own gather serves the last FFN's exit.""" - def build(self, exit_rows, *, disable_attn_tp_gather=False, gather=None): + class EveryRowRead(NormQuantReadout): + reads_after_attn_tp_gather = True + + def build( + self, + *, + final_read=None, + disable_attn_tp_gather=False, + gather=None, + read=None, + ): parallel = fixture.parallel_of( attn_dp=1, attn_tp=2, disable_attn_tp_gather=disable_attn_tp_gather ) + attn_read = {} if read is None else dict(read=read) with fixture.planning(parallel, a2a=True, boundary_reduction="ar"): - with layer_stack(): + with layer_stack(final_read=final_read): return [ stage for idx in range(2) for stage in append_stages( ( - declare_attn(attn_tp_gather=gather if idx else None), + declare_attn( + attn_tp_gather=gather if idx else None, **attn_read + ), fixture.Norm(), ), ( - declare_ffn( - sparse=True, - next_layer_sparse=True, - exit_rows=exit_rows, - ), + declare_ffn(sparse=True, next_layer_sparse=True), fixture.Norm(), ), ) ] - def test_the_last_ffn_stays_on_its_slice(self): - for exit_rows in (ExitRows.SLICE, None): - with self.subTest(exit_rows=exit_rows): - last = self.build(exit_rows)[-1].plan.paths[BatchVariant.ORDINARY] - if exit_rows is ExitRows.SLICE: - self.assertIs(last.output_move, keep_output) + def test_the_last_ffn_stays_on_its_slice_for_a_read_of_the_slice(self): + for reads_slices in (True, False): + with self.subTest(reads_slices=reads_slices): + final_read = SimpleNamespace(reads_attn_tp_slices=reads_slices) + last = self.build(final_read=final_read)[-1] + path = last.plan.paths[BatchVariant.ORDINARY] + if reads_slices: + self.assertIs(path.output_move, keep_output) else: - self.assertIs(last.output_move.func, update_attn_tp_gather_output) + self.assertIs(path.output_move.func, update_attn_tp_gather_output) + + def test_the_last_ffn_gathers_in_the_final_read_gather(self): + def gather(hidden_states): + return None + + last = self.build(final_read=SimpleNamespace(attn_tp_gather=gather))[-1] + move = last.plan.paths[BatchVariant.ORDINARY].output_move + self.assertIs(move.func, update_attn_tp_gather_output) + self.assertIs(move.keywords["gather"], gather) def test_an_unpadded_batch_leaves_the_last_ffn_on_the_attention_rows(self): - last = self.build(ExitRows.SLICE, disable_attn_tp_gather=True)[-1] + last = self.build( + final_read=SimpleNamespace(reads_attn_tp_slices=True), + disable_attn_tp_gather=True, + )[-1] unpadded = last.plan.paths[BatchVariant.UNPADDED] self.assertEqual(unpadded.output.layout.sharded, frozenset()) self.assertIs(unpadded.output_move, keep_output) @@ -507,10 +532,24 @@ class TestSliceExitRows(CustomTestCase): def gather(hidden_states): return None - _, _, attention, _ = self.build(ExitRows.SLICE, gather=gather) + first_ffn, attention = self.build(gather=gather)[1:3] move = attention.plan.paths[BatchVariant.ORDINARY].entry.input_move self.assertIs(move.func, attn_tp_gather_input) self.assertIs(move.keywords["gather"], gather) + path = first_ffn.plan.paths[BatchVariant.ORDINARY] + self.assertIs(path.output_move, keep_output) + + def test_a_read_of_every_row_has_the_ffn_gather_in_its_gather(self): + def gather(hidden_states): + return None + + first_ffn, attention = self.build(gather=gather, read=self.EveryRowRead())[1:3] + move = first_ffn.plan.paths[BatchVariant.ORDINARY].output_move + self.assertIs(move.func, update_attn_tp_gather_output) + self.assertIs(move.keywords["gather"], gather) + entry = attention.plan.paths[BatchVariant.ORDINARY].entry + self.assertIsNone(entry.input_move) + self.assertEqual(entry.input_rows.sharded, frozenset()) def test_a_read_that_gathers_needs_a_plain_add_or_a_written_stream(self): def gather_read(hidden_states, residual, norm): @@ -536,7 +575,7 @@ class TestSliceExitRows(CustomTestCase): with self.subTest(update=type(update).__name__): with ( fixture.planning(parallel, a2a=True, boundary_reduction="ar"), - layer_stack(), + layer_stack(final_read=SimpleNamespace(reads_attn_tp_slices=True)), ): stages = [ stage @@ -548,7 +587,6 @@ class TestSliceExitRows(CustomTestCase): sparse=True, next_layer_sparse=True, update=update, - exit_rows=ExitRows.SLICE, ), fixture.Norm(), ), diff --git a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py index 7ed43787bc..1b37db9469 100644 --- a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py @@ -1,5 +1,6 @@ -"""A stack read through the attention-residual bank's stage reads computes -what the decoder computed when its MLP folded the pending add itself.""" +"""A stack read through the attention-residual bank's stage reads matches a +reference that drives AttnResidual directly, with each MLP adding the pending +residual itself.""" import unittest from types import SimpleNamespace @@ -11,7 +12,6 @@ import torch from sglang.srt.layers import attn_residual from sglang.srt.layers.layer_boundary import ( BatchVariant, - ExitRows, ReadoutFusion, SumGroup, append_stages, @@ -85,7 +85,7 @@ class _Layer: def mlp(self, x): return torch.sin(x) * self.mlp_weight - def ops(self, bank): + def ops(self, bank, *, reads_slices=False): return AttnBankState( bank, self.attn_proj, @@ -93,6 +93,7 @@ class _Layer: self.ffn_proj, self.ffn_score_norm, writes_block=self.writes_block, + reads_slices=reads_slices, ).residual_ops() @@ -100,9 +101,9 @@ LAYER_LIST = [_Layer(i) for i in range(LAYERS)] OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM = _Proj(99), _Norm(99), _Norm(100) -def _legacy(hidden, layers, bank_rows=None, *, final=True): - """The decoder's own order: the MLP adds the pending prefix to its output, - so each layer hands on the whole head.""" +def _reference(hidden, layers, bank_rows=None, *, final=True): + """The reference order: the MLP adds the pending prefix to its output, so + each layer hands on the whole head.""" bank = attn_residual.AttnResidual( hidden, -(-(layers.stop) // BLOCK), block_residual=bank_rows ) @@ -193,8 +194,8 @@ class TestAttnBankResidualOps(CustomTestCase): def assert_identical(self, got, want): torch.testing.assert_close(got, want, rtol=0, atol=0) - def test_the_stack_matches_the_decoders_own_order(self): - want = _legacy(self.hidden, slice(0, LAYERS)) + def test_the_stack_matches_the_reference_order(self): + want = _reference(self.hidden, slice(0, LAYERS)) for read_after_write in (False, True): for writes_stream in ((), (1, 3), (LAYERS - 1,)): with self.subTest( @@ -209,7 +210,7 @@ class TestAttnBankResidualOps(CustomTestCase): self.assert_identical(got, want) def test_a_pipeline_rank_continues_from_the_head_and_the_bank(self): - want = _legacy(self.hidden, slice(0, LAYERS)) + want = _reference(self.hidden, slice(0, LAYERS)) # Split before a layer that writes a block and before one that reads, # after a layer that adds the residual itself and after one that # leaves the add to the next read. @@ -223,12 +224,12 @@ class TestAttnBankResidualOps(CustomTestCase): writes_stream=writes_stream, final=False, ) - legacy_head, legacy_rows = _legacy( + want_head, want_rows = _reference( self.hidden, slice(0, split), final=False ) - # The wire carries what the decoder's own order sent. - self.assert_identical(head, legacy_head) - self.assert_identical(bank_rows, legacy_rows) + # The wire carries what the reference order sends. + self.assert_identical(head, want_head) + self.assert_identical(bank_rows, want_rows) got = _staged( head, slice(split, LAYERS), @@ -346,9 +347,9 @@ class TestAttnBankSpMoeStages(CustomTestCase): """A latent MoE dispatched over an a2a backend with attention TP runs on this rank's shard of the rows: the FFN's entry reduce-scatters the attention output and slices the residual, and each MoE layer's exit - gathers its stream back to every row, as the decoder did.""" + gathers its stream back to every row, which the bank's next read needs.""" - def build(self, exit_rows): + def build(self, *, reads_slices=False): holder = AttnBank() with ( fixture.planning( @@ -371,12 +372,14 @@ class TestAttnBankSpMoeStages(CustomTestCase): sparse=True, next_layer_sparse=True, output_complete=True, - exit_rows=exit_rows, ), fixture.Norm(), ), )[1] - for ops in (LAYER_LIST[i].ops(holder) for i in range(3)) + for ops in ( + LAYER_LIST[i].ops(holder, reads_slices=reads_slices) + for i in range(3) + ) ] def entry_step(self, ffn): @@ -384,7 +387,7 @@ class TestAttnBankSpMoeStages(CustomTestCase): return prepare.keywords["step"] def test_each_moe_layer_returns_to_every_row(self): - for ffn in self.build(ExitRows.ATTENTION): + for ffn in self.build(): step = self.entry_step(ffn) self.assertIs(step.func, comm_ops._attn_tp_reduce_scatter_update_read) self.assertTrue(step.keywords["scatters_residual"]) diff --git a/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py index 1429481936..e939b7dff9 100644 --- a/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py +++ b/test/registered/unit/layer_boundary/test_kimi_k3_stage_kernels.py @@ -1,6 +1,6 @@ """Kimi K3's own kernels for a layer's sums and gathers serve its attention-residual bank only: the standard residual path keeps the -boundary's collectives, as the decoder did, even where those kernels run. +boundary's collectives, even where those kernels run. With SGLANG_K3_SP_ATTN_RES the bank and the stream stay on each rank's shard across SP-MoE layers.""" @@ -16,7 +16,10 @@ from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective from sglang.srt.layers.layer_boundary import BatchVariant, layer_stack from sglang.srt.layers.layer_boundary import prepare as comm_ops from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, keep_output -from sglang.srt.layers.layer_boundary.residual.attn_bank import AttnBank +from sglang.srt.layers.layer_boundary.residual.attn_bank import ( + AttnBank, + AttnBankOutputRead, +) from sglang.srt.models import kimi_k3 from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -53,11 +56,26 @@ def moe_layer(*, bank, sp_moe, all_reduce_fusion): def declare(layers, *, sp_moe, carries=False, config=CONFIG): """Declare ``layers`` (index -> layer) in one layer stack, as if the - stack carried the bank's shards (``carries``).""" + stack carried the bank's shards (``carries``). A bank stack ends on the + model's final read of the bank, which gathers an SP-MoE layer's rows in + K3's tuned all-gather.""" bank = AttnBank() + final_read = ( + AttnBankOutputRead( + bank, + None, + None, + fixture.Norm(), + attn_tp_gather=k3_sp_collective.all_gather if sp_moe else None, + reads_attn_tp_slices=carries, + ) + if any(layer.use_attn_residuals for layer in layers.values()) + else None + ) with ( patch.object(k3_sp_collective, "enabled", return_value=True), patch.object(k3_ar_fusion, "enabled", return_value=True), + patch.object(kimi_k3, "_shards_moe_rows", return_value=sp_moe), patch.object(kimi_k3, "_carries_bank_slices", return_value=carries), envs.SGLANG_K3_SP_ATTN_RES.override(carries), fixture.planning( @@ -65,7 +83,7 @@ def declare(layers, *, sp_moe, carries=False, config=CONFIG): a2a=sp_moe, boundary_reduction="ar", ), - layer_stack(), + layer_stack(final_read=final_read), ): for idx, layer in layers.items(): layer._declare_stages(config, idx, bank) diff --git a/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py index b11419434a..e0555bb020 100644 --- a/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py +++ b/test/registered/unit/layer_boundary/test_qwen4_exp_ple_rows.py @@ -9,14 +9,14 @@ from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.layers.layer_boundary.layout import TokenAxis from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState -from sglang.srt.models.qwen4_exp import _build_qwen4_exp_stages +from sglang.srt.models.qwen4_exp import _build_qwen4_exp_stages, _has_ple from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") -def _residual_ops(): +def _residual_ops(*, attn_reads_every_row): def unused(*args, **kwargs): raise AssertionError("construction only") @@ -26,6 +26,7 @@ def _residual_ops(): ffn_mix=unused, attn_combine=unused, ffn_combine=unused, + attn_reads_every_row=attn_reads_every_row, ).residual_ops() @@ -45,7 +46,8 @@ class TestQwen4ExpPleRows(CustomTestCase): ): layers = [ _build_qwen4_exp_stages( - _residual_ops(), sparse=True, layer_id=layer_id, config=config + _residual_ops(attn_reads_every_row=_has_ple(layer_id, config)), + sparse=True, ) for layer_id in range(config.num_hidden_layers) ] diff --git a/test/registered/unit/layers/test_kimi_attention_parallel_groups.py b/test/registered/unit/layers/test_kimi_attention_parallel_groups.py index d05c72892a..abdb7ac454 100644 --- a/test/registered/unit/layers/test_kimi_attention_parallel_groups.py +++ b/test/registered/unit/layers/test_kimi_attention_parallel_groups.py @@ -158,9 +158,8 @@ class TestKimiAttentionParallelGroups(CustomTestCase): self.assertEqual( module.all_reduce_fusion, active_fusion ) - self.assertEqual( - module.o_proj.reduce_results, not active_fusion - ) + # The stage boundary completes the attention-TP sum. + self.assertFalse(module.o_proj.reduce_results) self.assertEqual( module.o_proj.use_dp_attention_reduce, not active_fusion, diff --git a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py index 8ba7a73e72..56b1eb23c4 100644 --- a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py +++ b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py @@ -20,14 +20,13 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") -def build_mlp(group=None, *, width=16, quant_config=None, reduce=True): +def build_mlp(group=None, *, width=16, quant_config=None): kwargs = {} if group is None else dict(parallel_group=group) return kimi_k3.KimiK3MLP( width, 2 * width, "silu", quant_config=quant_config, - reduce_results=reduce, **kwargs, ) @@ -128,64 +127,60 @@ class TestKimiMLPParallelGroups(CustomTestCase): "replicated", "shared_experts_tp", ): - for reduce in (False, True): - module = build_mlp(group, reduce=reduce) - auto_attn = group is None and dense_attn and dp > 1 - r, s = ( - (0, 1) - if group == "replicated" - else (rank % 2, 2) - if group == "shared_experts_tp" - else (rank % (4 // dp), 4 // dp) - if group == "attn_tp" or auto_attn - else (rank, 4) - ) - self.assertEqual(module._dense_attn_tp, auto_attn) - for name in ("gate_up_proj", "down_proj"): - layer = getattr(module, name) - self.assertEqual(rank_size(layer), (r, s)) - expected, _ = load_projection(layer) - torch.testing.assert_close(layer.weight, expected) - x = ( - ( - torch.arange(2 * expected.shape[1]).reshape( - 2, -1 - ) - % 17 - - 8 + module = build_mlp(group) + auto_attn = group is None and dense_attn and dp > 1 + r, s = ( + (0, 1) + if group == "replicated" + else (rank % 2, 2) + if group == "shared_experts_tp" + else (rank % (4 // dp), 4 // dp) + if group == "attn_tp" or auto_attn + else (rank, 4) + ) + self.assertEqual(module._dense_attn_tp, auto_attn) + for name in ("gate_up_proj", "down_proj"): + layer = getattr(module, name) + self.assertEqual(rank_size(layer), (r, s)) + expected, _ = load_projection(layer) + torch.testing.assert_close(layer.weight, expected) + x = ( + ( + torch.arange(2 * expected.shape[1]).reshape( + 2, -1 ) - / 128 - ).to(layer.weight.dtype) - tp, attn = Mock(), Mock() - tp.all_reduce.side_effect = lambda v: v * 4 - attn.all_reduce.side_effect = lambda v: ( - v * (4 // dp) + % 17 + - 8 ) - row = name == "down_proj" - reference = F.linear(x, expected) - if row and reduce and s > 1: - reference *= (4 // dp) if auto_attn else 4 - with ( - parallel_scope(tp_group=tp, attn_tp_group=attn), - patch.object( - linear, - "use_symmetric_memory", - return_value=nullcontext(), - ) as allocation, - ): - actual = layer(x)[0] - torch.testing.assert_close(actual, reference) - if row: - self.assertEqual(layer.reduce_results, reduce) - self.assertEqual( - layer.use_dp_attention_reduce, auto_attn - ) - self.assertIs( - allocation.call_args.args[0], - attn if auto_attn else tp, - ) - else: - allocation.assert_not_called() + / 128 + ).to(layer.weight.dtype) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda v: v * 4 + attn.all_reduce.side_effect = lambda v: v * (4 // dp) + row = name == "down_proj" + reference = F.linear(x, expected) + with ( + parallel_scope(tp_group=tp, attn_tp_group=attn), + patch.object( + linear, + "use_symmetric_memory", + return_value=nullcontext(), + ) as allocation, + ): + actual = layer(x)[0] + torch.testing.assert_close(actual, reference) + if row: + # The stage boundary completes the sum. + self.assertFalse(layer.reduce_results) + self.assertEqual( + layer.use_dp_attention_reduce, auto_attn + ) + self.assertIs( + allocation.call_args.args[0], + attn if auto_attn else tp, + ) + else: + allocation.assert_not_called() def test_moe_shared_consumer_preserves_selection_and_execution_handle(self): for rank in range(4):