diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 06121eaecb..2969a9c5f7 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -200,7 +200,7 @@ hidden_states = residual_batch.final_norm( `skip_empty=True` still completes owed work on an empty batch, which may be a collective that other ranks join, and skips only the norm kernel. `final_norm()` and `to_pp()` release the batch's stream after they hand on its output. -This is a non-pipeline stack fragment. Pipeline reception uses the first boundary's `from_pp(tensors, forward_batch)` to reconstruct a stream; the sender uses `residual_batch.to_pp()`. A stream the producer already wrote, such as MHC's expanded streams, travels as `hidden_states` alone and arrives written. Dynamic completion work is resolved before transport. By default, `to_pp()` preserves a statically declared partial sum for the receiver's incoming contract to reconstruct and complete. Do not call `complete_output()` before that handoff: it would complete a sum that the receiver still expects to perform. +This is a non-pipeline stack fragment. Pipeline reception uses the first boundary's `from_pp(tensors, forward_batch)` to reconstruct a stream; the sender uses `residual_batch.to_pp()`. A stream the producer already wrote travels as `hidden_states` alone and arrives written: MHC's expanded streams, and the output of an FFN on its own rows (its attention-TP slice, or an unpadded batch's rows), which writes it at its exit before a pipeline handoff. The receiver knows this from the producer's declaration, not from which tensors arrive, since a captured CUDA graph's input buffers hold every key. Dynamic completion work is resolved before transport. By default, `to_pp()` preserves a statically declared partial sum for the receiver's incoming contract to reconstruct and complete. Do not call `complete_output()` before that handoff: it would complete a sum that the receiver still expects to perform. A terminal FFN normally completes its output before the final norm. The exception is a MoE finalize handoff. When the stage's fusion provider can defer the finalize at the terminal layer (for CuteDSL, `install_cutedsl_fusion(..., terminal_finalize=True)`), the exit may leave a `DeferredFinalize` for the final norm. Pass the handle straight to `residual_batch.final_norm(hidden_states, forward_batch, self.norm, finalize_norm=service)`, where `service` is the object `install_cutedsl_fusion()` returned; its `finalize(handoff, residual, gamma)` fuses finalize, all-reduce, residual add, and norm. The fused path reads `layernorm.gemma_weight`, so it currently expects a `GemmaRMSNorm` final norm. Without `finalize_norm`, `final_norm()` first completes the handoff unfused, and `capture` is then allowed. With `finalize_norm`, `final_norm()` rejects `capture` when a handoff arrives. Calling `complete_output()` first runs the producer's unfused finalize, and `snapshot()` raises on a handoff. See `python/sglang/srt/models/qwen3_5.py`. diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 6a9180f4de..e56027ab84 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -142,6 +142,8 @@ class StagePlan: is_branch: Branch reusing an already-read input; requires branch_input instead of a normal prepare to avoid repeating the read/update. terminal: Whether the stage ends the model's layer stack. + writes_at_handoff: Whether the stage, an FFN handing off to another + pipeline rank, writes its output into the residual at its exit. finishes_directly: Attention can publish its output with finish instead of an exit scope and output transport. qkv_latent_func: Optional hook for prepared attention input. @@ -164,6 +166,7 @@ class StagePlan: enters_stack=False, is_branch=False, terminal=False, + writes_at_handoff=False, finishes_directly=False, qkv_latent_func=None, fusions=None, @@ -217,6 +220,7 @@ class StagePlan: output_move=out.output_move, output_move_completes_sum=out.output_move_completes_sum, returns_over_dp=out.returns_over_dp, + writes_at_handoff=writes_at_handoff, ) @property @@ -304,6 +308,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options): enters_stack=incoming.producer is None, is_branch=declaration.prepared_from is not None, terminal=declaration.terminal, + writes_at_handoff=declaration.writes_at_handoff, finishes_directly=declaration.kind is StageKind.ATTENTION and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL, **options, diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 025ab6a89c..74c8a4c4c5 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -233,6 +233,8 @@ class StagePath(msgspec.Struct, frozen=True): batch by the attention-DP exit path. output_move_completes_sum: Whether that move also reduces the output. returns_over_dp: Whether output uses batch-dependent attention-DP transport. + writes_at_handoff: Whether the exit completes the output and writes it + into the residual, for an FFN that hands off to another pipeline rank. """ entry: EntryPath @@ -242,6 +244,7 @@ class StagePath(msgspec.Struct, frozen=True): output_move_completes_sum: bool = False returns_over_dp: bool = False + writes_at_handoff: bool = False class StageKind(Enum): diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index 35585fcd77..e0d7fcffe2 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -194,8 +194,10 @@ class ExitPolicy: self.plan.fusions is not None and self.plan.fusions.can_defer_finalize(self.plan, forward_batch) ) - if not steps.output.may_defer_to_next and not ( - self.plan.terminal and defer_moe_finalize + # An FFN that writes its output at a pipeline handoff completes it here. + if steps.writes_at_handoff or ( + not steps.output.may_defer_to_next + and not (self.plan.terminal and defer_moe_finalize) ): return ExitDecision( defer_moe_finalize=False, @@ -279,7 +281,7 @@ class ExitPolicy: forward_batch=forward_batch, ) update = steps.output.update - if residual is not None and update.applied_at_exit: + if residual is not None and (update.applied_at_exit or steps.writes_at_handoff): hidden_states = update.update(hidden_states, residual) residual = None return hidden_states, residual diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 9b8e4cfa4c..1a9045bd7b 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -77,6 +77,16 @@ def _row_layouts(variant): return axes, attention, local, Layout.sharded_over(axis_sizes=axes) +def _ffn_on_rank_rows(sparse, dense_tp_size) -> bool: + """Whether an FFN computes on this rank's own rows (its attention-TP slice + of them, or an unpadded batch's rows) rather than on the attention's.""" + if sparse: + return is_moe_input_scattered_across_dp_ranks() + if dense_tp_size is None: + return is_dense_ffn_fully_dp() + return dense_tp_size == 1 + + def _resolve_ffn( variant, *, @@ -100,11 +110,7 @@ def _resolve_ffn( use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output cp_shards = _prefill_cp_shards_tokens() axes, attention, local, full = _row_layouts(variant) - on_rank_rows = ( - is_moe_input_scattered_across_dp_ranks() - if sparse - else (is_dense_ffn_fully_dp() if dense_tp_size is None else dense_tp_size == 1) - ) + on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size) if parallel.attn_cp_size > 1 and sparse: _reject_unsupported_cp_moe(on_rank_rows, cp_shards) on_cp_shards = ( @@ -214,6 +220,8 @@ class StageDeclaration: dense_tp_size: Dense FFN compute width: None uses the configured width, 1 means local compute, and the full TP size means TP compute. exit_rows: Required FFN output rows at the layer or branch exit. + writes_at_handoff: Whether this FFN writes its output into the residual + at its exit because the next stage is on another pipeline rank. previous: Declaration whose output this stage consumes, as the stack records it; across pipeline ranks it is built locally. prepared_from: Declaration whose already-read input a branch reuses. @@ -233,6 +241,7 @@ class StageDeclaration: tp_group: SumGroup = SumGroup.ATTN_TP dense_tp_size: Optional[int] = None exit_rows: Optional[ExitRows] = None + writes_at_handoff: bool = False # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -722,10 +731,28 @@ class _Chain: __slots__ = ("previous", "pending") def __init__(self, previous): - self.previous = _detached(previous) + self.previous = _detached(_handed_off(previous)) self.pending = None +def _handed_off(declaration): + """A stage whose output crosses to another pipeline rank: an FFN leaves it + on the attention's rows, which is what the handoff carries and the next + rank's first stage reads, even where it would otherwise stay on this + rank's attention-TP slice of them. An FFN on its own rows also writes its + output into the residual there, on every batch, so the handoff carries the + written stream and the receiver knows it from this declaration.""" + if declaration is None or declaration.kind is not StageKind.FFN: + return declaration + return replace( + declaration, + exit_rows=ExitRows.ATTENTION, + writes_at_handoff=_ffn_on_rank_rows( + declaration.sparse, declaration.dense_tp_size + ), + ) + + def _detached(declaration): """The declaration a following append extends, without its own history. @@ -741,6 +768,10 @@ def _detached(declaration): def _bind_stack(appends, *, previous, following): """Bind every appended stage, in order, and fill in the boundaries each append returned.""" + if following is not None: + # The last stage hands off to the next rank. + last = next(a for a in reversed(appends) if a.prepared_from is None) + last.declarations[-1] = _handed_off(last.declarations[-1]) chain = _Chain(previous) # A returned declaration's boundary as bound, for the branches that read it. sources = {} diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 970de6a22e..66300f895f 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -223,12 +223,12 @@ class StageBoundary: Tensor or owed handle for prepare. A producer-written residual and a declared partial sum are reconstructed from the incoming contract. """ + previous = self.declaration.previous hidden_states, residual = from_pp( tensors, - residual_in_hidden=( - self.declaration.previous is not None - and self.declaration.previous.update.applied_at_exit - ), + # The producer's declaration says whether the stream arrives written. + residual_in_hidden=previous is not None + and (previous.update.applied_at_exit or previous.writes_at_handoff), allow_missing_residual=allow_missing_residual, ) declared_sum = self.entry(forward_batch).declared_sum diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index bf10734068..08f2c6d14b 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -1,7 +1,9 @@ """Small fixtures around production boundaries; no parallel layout recipes.""" from types import SimpleNamespace +from unittest import mock +from sglang.srt.layers.layer_boundary import factories from sglang.srt.layers.layer_boundary.construction import StagePlan from sglang.srt.layers.layer_boundary.factories import ( append_stages, @@ -21,15 +23,25 @@ def build_stages(*stages, previous=None, terminal=False): *stages: Items of (declaration, norm) or (declaration, norm, options), as for append_stages. previous: Declaration of the stage before the first one, standing for - a layer the test does not build; None starts the stack. + a layer on the same rank that the test does not build; None starts + the stack. terminal: Whether the last stage ends the model's layer stack; - otherwise the next layer's attention follows it. + otherwise the next layer's attention follows it on the same rank. """ - with layer_stack( - previous_layers=( - [lambda: append_stages((previous, None))] if previous is not None else [] + # The stack's neighbour hooks build layers another pipeline rank holds; + # here they stand for this rank's, so nothing is handed off to them. + with ( + mock.patch.object(factories, "_handed_off", lambda declaration: declaration), + layer_stack( + previous_layers=( + [lambda: append_stages((previous, None))] + if previous is not None + else [] + ), + next_layers=( + [] if terminal else [lambda: append_stages((declare_attn(), None))] + ), ), - next_layers=[] if terminal else [lambda: append_stages((declare_attn(), None))], ): return append_stages(*stages) diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index c0b67bdb3c..926436055d 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -1,20 +1,31 @@ """A layer stack connects the stages appended to it in order.""" import unittest +from functools import partial +from types import SimpleNamespace from unittest.mock import patch +import msgspec import test_declared_decoder_boundary as fixture import torch from torch import nn from sglang.srt.layers.layer_boundary import ( + BatchVariant, + ExitRows, ProducerReduction, append_stages, declare_attn, declare_ffn, layer_stack, ) +from sglang.srt.layers.layer_boundary.ops import ( + attn_tp_gather_input, + keep_output, + update_attn_tp_gather_output, +) from sglang.srt.layers.rotary_embedding import factory as rope_factory +from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.utils.common import is_building_neighbour_layer, make_layers from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -293,5 +304,98 @@ class TestMakeLayers(CustomTestCase): self.assertEqual(added, {("test", 0), ("test", 1)}) +class TestPipelineHandoff(CustomTestCase): + """With an a2a MoE over attention TP, an FFN leaves its output on this + rank's slice of the rows for the next layer to gather; across a pipeline + handoff it returns to the attention's rows, which both ranks agree on.""" + + def setUp(self): + self.planning = fixture.planning( + fixture.parallel_of(attn_dp=1, attn_tp=2), + a2a=True, + boundary_reduction="ar", + ) + self.planning.__enter__() + self.addCleanup(self.planning.__exit__, None, None, None) + + @staticmethod + def path(stage): + return stage.plan.paths[BatchVariant.ORDINARY] + + def test_the_sending_rank_returns_to_the_attention_rows(self): + with layer_stack(next_layers=[partial(layer, sparse=True)]): + first_attn, first_ffn, second_attn, second_ffn = [ + s for _ in range(2) for s in layer(sparse=True) + ] + # Within the rank the output stays on the slice, and the next + # attention gathers its input. + self.assertIs(self.path(first_ffn).output_move, keep_output) + self.assertIs(self.path(second_attn).entry.input_move, attn_tp_gather_input) + self.assertIs( + self.path(second_ffn).output_move.func, update_attn_tp_gather_output + ) + self.assertFalse(self.path(first_ffn).writes_at_handoff) + self.assertTrue(self.path(second_ffn).writes_at_handoff) + + def test_the_handoff_writes_what_the_move_leaves(self): + # A move that leaves the residual, as for an FFN on an unpadded batch's + # rows: the exit still writes the output into it. + with layer_stack(next_layers=[partial(layer, sparse=True)]): + ffn = [s for s in layer(sparse=True)][-1] + steps = msgspec.structs.replace(self.path(ffn), output_move=keep_output) + hidden, residual = torch.ones(4, 4), torch.full((4, 4), 2.0) + want = hidden + residual + written, left = ffn.plan.output._complete_now( + hidden, + residual, + forward_batch=None, + dp_step=None, + steps=steps, + output_move=keep_output, + owes_sum=False, + ) + self.assertIsNone(left) + torch.testing.assert_close(written, want) + + def test_the_receiving_rank_reads_the_attention_rows(self): + with layer_stack(previous_layers=[partial(layer, sparse=True)]): + attention, _ = layer(sparse=True) + self.assertIsNone(self.path(attention).entry.input_move) + self.assertIs(attention.declaration.previous.exit_rows, ExitRows.ATTENTION) + + def test_the_receiving_rank_reads_the_stream_as_written(self): + # A residual beside it, as a captured graph's input buffer holds, is + # not read. + with layer_stack(previous_layers=[partial(layer, sparse=True)]): + attention, _ = layer(sparse=True) + attention.plan.path_for = lambda _: self.path(attention) + hidden, stale = torch.ones(4, 4), torch.full((4, 4), 2.0) + for wire in ( + {"hidden_states": hidden}, + {"hidden_states": hidden, "residual": stale}, + ): + with self.subTest(keys=sorted(wire)): + batch = SimpleNamespace(residual_stream=None) + attention.from_pp(PPProxyTensors(wire), batch) + self.assertIs(batch.residual_stream.residual, hidden) + self.assertIsNone(batch.residual_stream.pending) + + def test_a_handoff_from_the_attention_rows_carries_its_residual(self): + # A dense FFN over the TP group computes on the attention's rows and + # leaves the residual add to the receiver, which requires it. + with layer_stack(previous_layers=[partial(layer, sparse=False)]): + attention, _ = layer(sparse=False) + self.assertFalse(attention.declaration.previous.writes_at_handoff) + attention.plan.path_for = lambda _: self.path(attention) + hidden, residual = torch.ones(4, 4), torch.full((4, 4), 2.0) + batch = SimpleNamespace(residual_stream=None) + wire = PPProxyTensors({"hidden_states": hidden, "residual": residual}) + attention.from_pp(wire, batch) + self.assertIs(batch.residual_stream.residual, residual) + self.assertIs(batch.residual_stream.pending.value, hidden) + with self.assertRaises(KeyError): + attention.from_pp(PPProxyTensors({"hidden_states": hidden}), batch) + + if __name__ == "__main__": unittest.main()