From 2b469040d2cc466726565c4ae203dde2bd802f9b Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:27:00 -0700 Subject: [PATCH] [Refactor] Support attention-TP dense FFNs and unpadded batches on stage boundaries (#42897) --- docs/docs/developer_guide/layer_boundary.mdx | 4 +- .../layer_boundary/adapters/attention.py | 14 +++- .../srt/layers/layer_boundary/construction.py | 25 ++++++ .../srt/layers/layer_boundary/contracts.py | 3 + .../srt/layers/layer_boundary/factories.py | 54 +++++++++--- .../srt/layers/layer_boundary/layout.py | 12 +++ python/sglang/srt/models/kimi_k3.py | 15 +--- python/sglang/test/boundary_fixtures.py | 1 + .../unit/layer_boundary/test_append_stages.py | 82 +++++++++++++++++++ .../test_declared_attention_cp.py | 1 + .../test_declared_decoder_boundary.py | 1 + .../test_independent_stage_construction.py | 6 ++ .../test_longcat_flash_shortcut.py | 7 +- .../layer_boundary/test_nemotron_h_stages.py | 2 + 14 files changed, 200 insertions(+), 27 deletions(-) diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 94643cece3..a397cdf50d 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -262,7 +262,7 @@ Common errors and their usual cause: | `start the layer stack before entering a stage` | The forward path, a split-prefill segment, or an MTP head skipped `residual_batch.start()` | | `output does not belong to this residual stream; change layer outputs through boundary accessors` | The model replaced a layer's output object (for example rebinding it to a new tensor) or passed another microbatch's output. This check compares object identity, so it does not catch in-place changes; the interface still forbids modifying layer outputs in place | | `write the residual update before taking the final output` | `take_output()` ran while a contribution was still pending; fold or norm it first | -| `no stage boundary path for the active batch` | The active batch variant (CP, input-scattered, sequence parallel) is not supported by this stage; add a server-argument check that rejects the configuration | +| `no stage boundary path for the active batch` | The active batch variant (CP, input-scattered, sequence parallel, unpadded) is not supported by this stage; add a server-argument check that rejects the configuration | | `append_stages needs an open layer stack` | The layers are built outside `make_layers()`; wrap their construction in `layer_stack()` | | `a prepared branch must enter through branch_input, not prepare` | A `prepared_from` stage was entered with `prepare()` | | `snapshot requires a plain residual update` | A snapshot was taken on a nonlinear (for example MHC) update | @@ -323,7 +323,7 @@ Models normally use factories, `StageBoundary`, and `residual.batch`. Keep row a ### Batch variants -`StagePlan` preconstructs ordinary, CP, input-scattered, and sequence-parallel variants when supported by the configured topology. A forward selects a path from active runtime facts. An absent active variant raises an error rather than silently using ordinary token rows. +`StagePlan` preconstructs ordinary, CP, input-scattered, sequence-parallel, and unpadded variants when supported by the configured topology. The unpadded variant serves batches whose rows do not divide over attention TP, which `--disable-attn-tp-gather` lets through without attention DP: an FFN that would run on this rank's attention-TP slice stays on the attention's rows instead. A forward selects a path from active runtime facts. An absent active variant raises an error rather than silently using ordinary token rows. Configuration validation rejects unsupported model/parallel combinations. Boundary construction also rejects transitions without a correct implementation. Adding an enum value or a layout does not by itself provide a new collective path. diff --git a/python/sglang/srt/layers/layer_boundary/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py index a53ad65e13..eee81fec68 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -26,6 +26,7 @@ from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) from sglang.srt.layers.layer_boundary.layout import ( + batches_are_unpadded, is_dense_ffn_fully_dp, ) from sglang.srt.layers.moe import get_moe_a2a_backend @@ -159,9 +160,16 @@ def get_attn_tp_context(): def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor: - gathered = get_local_dp_buffer( - get_parallel().attn_tp_group, hidden_size=tensor.shape[-1] - ) + parallel = get_parallel() + if batches_are_unpadded(): + # MLP sync, which sizes the local DP buffer for a batch, does not run. + gathered = tensor.new_empty( + (tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1]) + ) + else: + gathered = get_local_dp_buffer( + parallel.attn_tp_group, hidden_size=tensor.shape[-1] + ) attn_tp_all_gather_into_tensor(gathered, tensor) return gathered diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index e56027ab84..ae7791994f 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import ( _batch_shards_over_cp, _cp_gathers_over_attn_cp, _prefill_cp_shards_tokens, + batches_are_unpadded, is_dense_ffn_fully_dp, ) from sglang.srt.layers.layer_boundary.prepare import ( @@ -53,6 +54,7 @@ from sglang.srt.layers.layer_boundary.prepare import ( from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.layers.moe import ( get_moe_a2a_backend, + is_moe_input_scattered_across_dp_ranks, ) from sglang.srt.runtime_context import ( get_forward, @@ -91,6 +93,20 @@ def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None ) +def _unpadded_possible() -> bool: + """Whether a batch whose rows do not divide over attention TP may reach an + FFN that would run on this rank's attention-TP slice, so that FFN needs a + variant that stays on the attention's rows.""" + return batches_are_unpadded() and ( + is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp() + ) + + +def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool: + """Whether this batch arrived with rows that do not divide over attention TP.""" + return forward_batch.input_ids.shape[0] % attn_tp_size != 0 + + def _input_scattered_possible() -> bool: """Whether a batch may run this layer with input-scattered attention: configured, on TP without attention DP, a prefill CP, an a2a backend or @@ -173,6 +189,11 @@ class StagePlan: ): self.norm = norm self.edges = dict(variants) + # The attention TP size an unpadded batch's rows are checked against, + # when such a batch may arrive (see BatchVariant.UNPADDED). + self._unpadded_attn_tp_size = ( + get_parallel().attn_tp_size if BatchVariant.UNPADDED in self.edges else None + ) self.enters_stack = enters_stack self.terminal = terminal self.finishes_directly = finishes_directly @@ -240,6 +261,10 @@ class StagePlan: return BatchVariant.INPUT_SCATTERED if _batch_shards_over_cp(forward_batch): return BatchVariant.CONTEXT_PARALLEL + if self._unpadded_attn_tp_size is not None and _rows_indivisible_over_attn_tp( + forward_batch, self._unpadded_attn_tp_size + ): + return BatchVariant.UNPADDED return BatchVariant.ORDINARY def path_for(self, forward_batch): diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 74c8a4c4c5..c194562daa 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -52,6 +52,9 @@ class BatchVariant(Enum): CONTEXT_PARALLEL = auto() INPUT_SCATTERED = auto() SEQUENCE_PARALLEL = auto() + # A batch whose rows do not divide over attention TP, which only arrives + # unpadded (--disable-attn-tp-gather without attention DP). + UNPADDED = auto() class InputContract(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index ae8add4bf8..4c1ebb8784 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -22,6 +22,7 @@ from sglang.srt.layers.layer_boundary.construction import ( _bind_stage, _input_scattered_possible, _reject_unsupported_cp_moe, + _unpadded_possible, _use_ag_after_qlora, ) from sglang.srt.layers.layer_boundary.contracts import ( @@ -64,6 +65,8 @@ def _active_variants(): yield BatchVariant.INPUT_SCATTERED if layernorm_sp.layernorm_sp_enabled(): yield BatchVariant.SEQUENCE_PARALLEL + if _unpadded_possible(): + yield BatchVariant.UNPADDED def _row_layouts(variant): @@ -99,8 +102,11 @@ def _resolve_ffn( output_complete=False, ): parallel = get_parallel() - if dense_tp_size not in (None, 1, parallel.tp_size): - raise ValueError("dense FFN rows support local compute or the full TP group") + if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size): + raise ValueError( + "dense FFN rows support local compute, the attention TP group or the " + "full TP group" + ) strategy = get_exec().comm.boundary_reduction if strategy not in ("ar", "rs", "rsv", "rs+rsv"): raise ValueError( @@ -114,13 +120,36 @@ def _resolve_ffn( on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size) if parallel.attn_cp_size > 1 and sparse: _reject_unsupported_cp_moe(on_rank_rows, cp_shards) + if variant is BatchVariant.UNPADDED and on_rank_rows: + # Rows that do not divide over attention TP stay whole: the FFN runs on + # the attention's rows (an a2a MoE dispatches them from every + # attention-TP rank) and its output is complete, as a2a's combine or + # local compute leaves it. + return ( + StageContract( + InputContract(attention, read=read), + OutputContract(attention, update=update, transform=output_transform), + ), + attention, + attention, + ) on_cp_shards = ( sparse and parallel.attn_cp_size > 1 and parallel.moe_dp_size == parallel.attn_cp_size and not _cp_gathers_over_attn_cp() ) - group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP + # Over attention TP, short of the full TP group: the FFN owes only the + # attention-TP sum, and nothing moves over DP. + on_attention_rows = ( + not sparse + and dense_tp_size == parallel.attn_tp_size + and dense_tp_size not in (1, parallel.tp_size) + ) + if sparse: + group = SumGroup.MOE_OUTPUT + else: + group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP if variant is BatchVariant.SEQUENCE_PARALLEL: return ( StageContract( @@ -160,13 +189,16 @@ def _resolve_ffn( may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp_shards else: may_leave = may_scatter = True - rows = ( - local - if on_rank_rows - else Layout.sharded_over( + if on_rank_rows: + rows = local + elif on_attention_rows: + rows = attention + # The output is already on the residual's rows: nothing to scatter. + use_reduce_scatter = use_reduce_scatterv = False + else: + rows = Layout.sharded_over( *((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes ) - ) # An FFN that writes the next stream itself hands on a complete output. complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete produced = ( @@ -219,7 +251,9 @@ class StageDeclaration: tp_group: Group partitioning attention heads. Full TP consumers require full token rows, including when TP shares the prefill CP group. dense_tp_size: Dense FFN compute width: None uses the configured width, - 1 means local compute, and the full TP size means TP compute. + 1 means local compute, the attention TP size means compute on the + attention's rows over attention TP, and the full TP size means TP + compute. exit_rows: Required FFN output rows at the layer or branch exit. writes_at_handoff: Whether this FFN writes its output into the residual at its exit because the next stage is on another pipeline rank. @@ -346,7 +380,7 @@ def declare_ffn( next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only to derive the TBO exit rows when exit_rows is not supplied. dense_tp_size: Dense compute width: None for configuration, 1 for local - compute, or the full TP size. + compute, the attention TP size, or the full TP size. exit_rows: Explicit output-row requirement; otherwise derived from the adjacent FFN kinds and TBO configuration. output_complete: Whether the compute completes its own output sum, diff --git a/python/sglang/srt/layers/layer_boundary/layout.py b/python/sglang/srt/layers/layer_boundary/layout.py index 5b3c491db0..32816cadc2 100644 --- a/python/sglang/srt/layers/layer_boundary/layout.py +++ b/python/sglang/srt/layers/layer_boundary/layout.py @@ -83,6 +83,18 @@ def is_dense_ffn_fully_dp(): return get_parallel().moe_dense_tp_size == 1 +def batches_are_unpadded() -> bool: + """Whether batches may reach the stages without padding to a multiple of + attention TP: --disable-attn-tp-gather skips that padding unless attention + DP is on.""" + parallel = get_parallel() + return ( + parallel.attn_tp_size > 1 + and not parallel.attn_dp_enabled + and parallel.disable_attn_tp_gather + ) + + def _prefill_cp_shards_tokens() -> bool: """Whether the strategy prefill CP path shards prefill tokens across CP ranks.""" parallel = get_parallel() diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 918d6369ca..ba0aba9a24 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -149,7 +149,6 @@ from sglang.srt.utils.common import ( add_prefix, get_bool_env_var, rank0_log, - require_mlp_sync, set_weight_attrs, ) @@ -2448,20 +2447,14 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool: def _uses_stage_boundaries(config: KimiLinearConfig) -> bool: """Whether the layers build stage boundaries, which is the same for every - layer of a stack. The layer's own communication still runs for: a dense - MLP sharded over attention TP; an attention-residual bank whose o_proj - all-reduce is fused with the pending add, or whose MoE on its attention-TP - token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded - carry also needs; and SP-MoE on batches that are not padded to a multiple - of attention TP (--disable-attn-tp-gather).""" - if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled(): - return False + layer of a stack. The layer's own communication still runs for: an + attention-residual bank whose o_proj all-reduce is fused with the pending + add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned + SP collectives, which the sharded carry also needs.""" if _fuses_attn_all_reduce(config): return False if not _shards_moe_rows(): return True - if not require_mlp_sync(): - return False return config.attn_res_block_size is None or not k3_sp_collective.enabled() diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 08f2c6d14b..33d7ccd43e 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -95,6 +95,7 @@ def stub_plan(): plan.fusions = None plan._publish_lora_layout = False plan._next_input_rows = None + plan._unpadded_attn_tp_size = None plan.paths = {} plan.enters_stack = False return plan diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index 926436055d..e7c9083f44 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -19,6 +19,8 @@ from sglang.srt.layers.layer_boundary import ( declare_ffn, layer_stack, ) +from sglang.srt.layers.layer_boundary import prepare as boundary_prepare +from sglang.srt.layers.layer_boundary.layout import SumGroup, TokenAxis from sglang.srt.layers.layer_boundary.ops import ( attn_tp_gather_input, keep_output, @@ -397,5 +399,85 @@ class TestPipelineHandoff(CustomTestCase): attention.from_pp(PPProxyTensors({"hidden_states": hidden}), batch) +class TestUnpaddedBatches(CustomTestCase): + """Without attention DP, --disable-attn-tp-gather lets a batch reach the + stages with rows that do not divide over attention TP. Such a batch keeps + an FFN that would run on this rank's attention-TP slice on the + attention's rows instead.""" + + def build(self, parallel, *, sparse): + with fixture.planning(parallel, a2a=True, boundary_reduction="ar"): + with layer_stack(): + return [s for _ in range(2) for s in layer(sparse=sparse)] + + def test_an_unpadded_batch_keeps_the_attention_rows(self): + for sparse in (True, False): + with self.subTest(sparse=sparse): + parallel = fixture.parallel_of( + attn_dp=1, + attn_tp=2, + disable_attn_tp_gather=True, + moe_dense_tp_size=None if sparse else 1, + ) + _, ffn, attention, _ = self.build(parallel, sparse=sparse) + ordinary = ffn.plan.paths[BatchVariant.ORDINARY] + unpadded = ffn.plan.paths[BatchVariant.UNPADDED] + # Rows that divide take the slice; the others complete the + # attention's sum on every row and hand on a complete output. + self.assertIn(TokenAxis.ATTN_TP, ordinary.entry.input_rows.sharded) + self.assertEqual(unpadded.entry.input_rows.sharded, frozenset()) + self.assertIs( + unpadded.entry.prepare.keywords["step"].func, + boundary_prepare._reduce_update_read, + ) + self.assertIsNone(unpadded.output.group) + self.assertIs(unpadded.output_move, keep_output) + # The next attention reads those rows as they are. + self.assertIs( + attention.plan.paths[BatchVariant.ORDINARY].entry.input_move, + attn_tp_gather_input, + ) + self.assertIsNone( + attention.plan.paths[BatchVariant.UNPADDED].entry.input_move + ) + + def test_padded_batches_have_no_unpadded_path(self): + for attn_dp, disabled in ((2, True), (1, False)): + with self.subTest(attn_dp=attn_dp, disable_attn_tp_gather=disabled): + parallel = fixture.parallel_of( + attn_dp=attn_dp, attn_tp=2, disable_attn_tp_gather=disabled + ) + _, ffn, _, _ = self.build(parallel, sparse=True) + self.assertNotIn(BatchVariant.UNPADDED, ffn.plan.paths) + + +class TestDenseFfnOverAttentionTp(CustomTestCase): + """A dense FFN sharded over attention TP under attention DP computes on + the attention's rows and sums over attention TP, with no DP move.""" + + def test_the_ffn_stays_on_the_attention_rows(self): + parallel = fixture.parallel_of(attn_dp=2, attn_tp=2) + with fixture.planning(parallel, boundary_reduction="ar"), layer_stack(): + stages = [ + stage + for _ in range(2) + for stage in append_stages( + (declare_attn(), fixture.Norm()), + (declare_ffn(dense_tp_size=2), fixture.Norm()), + ) + ] + for ffn in stages[1::2]: + path = ffn.plan.paths[BatchVariant.ORDINARY] + # The attention's sum completes on this rank's rows, ungathered. + self.assertIs( + path.entry.prepare.keywords["step"].func, + boundary_prepare._reduce_update_read, + ) + self.assertEqual(path.output.layout.sharded, {TokenAxis.ATTN_DP}) + self.assertIs(path.output.group, SumGroup.ATTN_TP) + self.assertFalse(path.returns_over_dp) + self.assertIs(path.output_move, keep_output) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 4b5d55596f..499cd80f2a 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -103,6 +103,7 @@ class TestAttentionCpBoundary(CustomTestCase): moe_tp_size=moe_tp_size, dwdp_size=1, enable_attn_tp_input_scattered=False, + disable_attn_tp_gather=False, tp_group=group("tp", range(tp_size)), attn_tp_group=group("attn_tp", range(tp_rank, tp_rank + attn_tp_size)), attn_cp_group=group("attn_cp", range(0, tp_size, attn_tp_size)), diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 776ab71160..77d81d8151 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -85,6 +85,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): dwdp_size=1, attn_dp_enabled=attn_dp > 1, enable_attn_tp_input_scattered=False, + disable_attn_tp_gather=False, tp_group=SimpleNamespace( name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp)) ), diff --git a/test/registered/unit/layer_boundary/test_independent_stage_construction.py b/test/registered/unit/layer_boundary/test_independent_stage_construction.py index 459ae93368..d16c06ea9b 100644 --- a/test/registered/unit/layer_boundary/test_independent_stage_construction.py +++ b/test/registered/unit/layer_boundary/test_independent_stage_construction.py @@ -39,6 +39,7 @@ class TestIndependentStageConstruction(CustomTestCase): for variant in BatchVariant: plan = stub_plan() + plan._unpadded_attn_tp_size = 2 ordinary = object() plan.paths[BatchVariant.ORDINARY] = ordinary with ( @@ -62,6 +63,11 @@ class TestIndependentStageConstruction(CustomTestCase): "_batch_shards_over_cp", return_value=variant is BatchVariant.CONTEXT_PARALLEL, ), + patch.object( + construction, + "_rows_indivisible_over_attn_tp", + return_value=variant is BatchVariant.UNPADDED, + ), ): if variant is BatchVariant.ORDINARY: self.assertIs(plan.path_for(None), ordinary) diff --git a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py index bc972bafbc..8feb763289 100644 --- a/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py +++ b/test/registered/unit/layer_boundary/test_longcat_flash_shortcut.py @@ -78,7 +78,12 @@ class TestLongcatShortcut(CustomTestCase): ), patch_communicator( "get_parallel", - return_value=SimpleNamespace(attn_tp_group=object()), + return_value=SimpleNamespace( + attn_tp_group=object(), + attn_tp_size=tp, + attn_dp_enabled=False, + disable_attn_tp_gather=False, + ), ), ): hidden, _ = layer( diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py index 6930d9ca81..f4b9f6a038 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py @@ -66,12 +66,14 @@ def stages(pattern, *, dp=1, tp=1, a2a=False): parallel = SimpleNamespace( attn_dp_size=dp, + attn_dp_enabled=dp > 1, attn_tp_size=tp, attn_cp_size=1, tp_size=dp * tp, moe_dp_size=1, moe_dense_tp_size=None, enable_attn_tp_input_scattered=False, + disable_attn_tp_gather=False, enable_prefill_cp=False, ) with (