diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index a987c2ff88..78f43ba607 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1661,6 +1661,70 @@ class LayerCommunicator: result as a context manager around the FFN call, then call ``finish``.""" return FfnExit(self, forward_batch) + def _branch_rows( + self, forward_batch: ForwardBatch + ) -> Tuple[Layout, Layout, Layout]: + """The rows of this layer's FFN input, of its residual while the FFN + runs, and of what the layer hands on, for a batch that runs its ordinary + steps.""" + if self._batch_steps(forward_batch) is not self._steps: + raise NotImplementedError( + "a branch on a batch with attention CP, LayerNorm SP or " + "input-scattered attention" + ) + if self._declared is not None: + sides = self._declared + return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows + layouts, modes = self._context.layouts, self.layer_scatter_modes + return ( + layouts[modes.mlp_mode], + layouts[modes.middle_residual_mode], + layouts[modes.layer_output_mode], + ) + + def branch_input( + self, + source: "LayerCommunicator", + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """The FFN input and residual that ``source``'s boundary read for its + own FFN, for this layer's FFN, which branches from the same input: moved + to the rows this FFN needs and its residual's rows.""" + rows, residual_rows, _ = source._branch_rows(forward_batch) + to, residual_to, _ = self._branch_rows(forward_batch) + return ( + move_rows(hidden_states, rows, to, forward_batch), + move_rows(residual, residual_rows, residual_to, forward_batch), + ) + + def branch_output( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + """This layer's complete FFN output as a branch's contribution, which + adds to the layer's output without writing the residual: moved to the + rows the layer hands on.""" + rows, _, to = self._branch_rows(forward_batch) + return move_rows(hidden_states, rows, to, forward_batch) + + def merge_branch( + self, + contribution: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor, + source: "LayerCommunicator", + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A contribution from ``branch_output`` summed with what ``source``'s + layer hands on, ``hidden_states`` and ``residual``, moved to the rows + this layer hands on.""" + _, _, rows = source._branch_rows(forward_batch) + _, _, to = self._branch_rows(forward_batch) + hidden_states = move_rows(hidden_states, rows, to, forward_batch) + residual = move_rows(residual, rows, to, forward_batch) + return contribution + hidden_states, residual + def finish_layer_stack( self, hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], @@ -2139,6 +2203,34 @@ def _reduce_and_redistribute_output_to_dp( return global_hidden_states +def move_rows( + hidden_states: torch.Tensor, + rows: Layout, + to: Layout, + forward_batch: ForwardBatch, +) -> torch.Tensor: + """A complete value from the rows it is on to ``to``: gathered over the + token axes ``to`` does not shard (attention TP, then attention DP), or cut to + this rank's share of those it does (attention DP, then attention TP).""" + gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded + if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut: + raise NotImplementedError(f"{rows=} {to=}") + if TokenAxis.ATTN_TP_SCATTER in gathered: + hidden_states = _redistribute_from_attn_tp_shards(hidden_states) + if TokenAxis.ATTN_DP in gathered: + hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch) + if TokenAxis.ATTN_DP in cut: + hidden_states = _to_local_tokens( + _redistribute_output, forward_batch, hidden_states + ) + if TokenAxis.ATTN_TP_SCATTER in cut: + parallel = get_parallel() + hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[ + parallel.attn_tp_rank + ] + return hidden_states + + class FusedMlpInput(msgspec.Struct, frozen=True): """A kernel that completes the sum the attention output owes together with the residual add and the post-attention norm, in that order. diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 353a560ae4..61f77a5632 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -90,9 +90,8 @@ from sglang.srt.model_loader.utils import ( ) from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA -from sglang.srt.runtime_context import get_parallel -from sglang.srt.runtime_context import get_parallel as _gp from sglang.srt.runtime_context import ( + get_parallel, get_stream, ) from sglang.srt.utils import ( @@ -140,23 +139,6 @@ _LONGCAT_FLASH_ROUTER_HPC_GEMM_MIN_M = { } -def _scmoe_align_rows(t, target): - """Align a [rows,H] tensor to `target` rows across the attn-tp group: - all_gather when target>rows (target==rows*attn_tp_size), or take this rank's - contiguous segment when target cur: - out = t.new_empty((target, *t.shape[1:])) - _ag(out, t.contiguous()) - return out - r = _gp().attn_tp_rank - return t[r * target : r * target + target].contiguous() - - class LongcatFlashMLP(nn.Module): def __init__( self, @@ -420,6 +402,7 @@ class LongcatFlashDecoderLayer(nn.Module): if "mlps" in getattr(config, "disable_quant_module", []) else quant_config ), + reduce_results=True, prefix=add_prefix(f"mlps.{i}", prefix), ) for i in range(2) @@ -453,6 +436,7 @@ class LongcatFlashDecoderLayer(nn.Module): input_layernorm=self.input_layernorm[i], post_attention_layernorm=self.post_attention_layernorm[i], qkv_latent_func=self.self_attn[i].prepare_qkv_latent, + allow_deferred_ffn_reduction=False, ) for i in range(2) ] @@ -502,12 +486,8 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states, residual = self.moe_layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) - moe_hidden_states = hidden_states.clone() - # The final gather adds its residual; the dense branch already carries it. - moe_residual = torch.zeros_like(residual) - moe_hidden_states = self.mlp(moe_hidden_states) - moe_hidden_states, moe_residual = self.moe_layer_communicator.postprocess_layer( - moe_hidden_states, moe_residual, forward_batch + moe_hidden_states = self.moe_layer_communicator.branch_output( + self.mlp(hidden_states.clone()), forward_batch ) hidden_states, residual, prev_topk_indices = self.forward_mlp( @@ -519,12 +499,13 @@ class LongcatFlashDecoderLayer(nn.Module): prev_topk_indices, ) - # SCMOE_ATTN1_GATHER: reconcile mlp-branch (hidden/residual) with moe-branch - # (moe_hidden_states) row counts before the add, so residual tracks hidden. - _scmoe_tgt = moe_hidden_states.shape[0] - hidden_states = _scmoe_align_rows(hidden_states, _scmoe_tgt) - residual = _scmoe_align_rows(residual, _scmoe_tgt) - hidden_states = moe_hidden_states + hidden_states + hidden_states, residual = self.moe_layer_communicator.merge_branch( + moe_hidden_states, + hidden_states, + residual, + self.mlp_layer_communicator[1], + forward_batch, + ) return hidden_states, residual, prev_topk_indices def forward_mlp( @@ -536,25 +517,13 @@ class LongcatFlashDecoderLayer(nn.Module): zero_allocator, prev_topk_indices, ): - # SCMOE_ATTN1_GATHER: gather the dense-branch hidden(+residual) to full - # tokens before mlps[0] so its all_reduce is valid; slice back at the merge. - from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _scmoe_gab - - _scmoe_ats = self.attn_tp_size - _scmoe_do = ( - not _scmoe_gab().is_none() - and _scmoe_ats > 1 - and hidden_states.shape[0] != 0 - and hidden_states.shape[0] * _scmoe_ats == positions.shape[0] + # first_mlp, on the input the MoE's boundary read + hidden_states, residual = self.mlp_layer_communicator[0].branch_input( + self.moe_layer_communicator, hidden_states, residual, forward_batch ) - if _scmoe_do: - hidden_states = _scmoe_align_rows(hidden_states, positions.shape[0]) - residual = _scmoe_align_rows(residual, positions.shape[0]) - - # first_mlp - hidden_states = self.mlps[0](hidden_states) - # TP all_reduce - hidden_states = tensor_model_parallel_all_reduce(hidden_states) + with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.mlps[0](hidden_states) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) # second_attn hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn( @@ -577,13 +546,9 @@ class LongcatFlashDecoderLayer(nn.Module): hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp( hidden_states, residual, forward_batch ) - hidden_states = self.mlps[1](hidden_states) - # TP all_reduce - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - - hidden_states, residual = self.mlp_layer_communicator[1].postprocess_layer( - hidden_states, residual, forward_batch - ) + with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.mlps[1](hidden_states) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual, prev_topk_indices diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index c25628b045..04decb74df 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -446,9 +446,11 @@ class TestSelectFfnCompletion(CustomTestCase): SRT_DIR = Path(sglang.__file__).resolve().parent / "srt" # Attributes a decoder layer holds its FFN, or parts of it, in. A layer that is -# one FFN stage holds it in ``mixer``. +# one FFN stage holds it in ``mixer``; a layer with several dense FFNs, in +# ``mlps``. FFN_ATTRS = { "mlp", + "mlps", "moe", "shared_expert", "shared_experts", @@ -493,17 +495,34 @@ class SourceIndex: def submodule_constructions(cls): - """(attribute, class name, call) for each `self.attr = Name(...)`.""" + """(attribute, class name, call) for each `self.attr = Name(...)`, and for + each `Name(...)` in `self.attr = nn.ModuleList([...])`.""" for node in ast.walk(cls): if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): - func = node.value.func for target in node.targets: if ( - isinstance(func, ast.Name) - and isinstance(target, ast.Attribute) + isinstance(target, ast.Attribute) and getattr(target.value, "id", None) == "self" ): - yield target.attr, func.id, node.value + for call in constructed(node.value): + yield target.attr, call.func.id, call + + +def constructed(call): + """The `Name(...)` calls a call builds: itself, or the elements of the list + an `nn.ModuleList` holds.""" + if isinstance(call.func, ast.Name): + yield call + elif getattr(call.func, "attr", None) == "ModuleList" and call.args: + items = call.args[0] + elements = ( + [items.elt] + if isinstance(items, ast.ListComp) + else getattr(items, "elts", []) + ) + for element in elements: + if isinstance(element, ast.Call) and isinstance(element.func, ast.Name): + yield element def attention_tp_reductions(cls): @@ -597,6 +616,10 @@ class TestFfnExitUsersOweATpSum(CustomTestCase): self.assertLessEqual( {(nemotron, "NemotronHMLP"), (nemotron, "NemotronHMoE")}, ffn_classes ) + # A layer with two dense FFNs holds them in an nn.ModuleList. + self.assertIn( + ("sglang.srt.models.longcat_flash", "LongcatFlashMLP"), ffn_classes + ) self.assertEqual(sorted(set(offenders)), []) diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index ff1f6b0646..563c2a0b1a 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1899,6 +1899,146 @@ class TestPrefillCP(CustomTestCase): self.assertIs(communicator._batch_steps(fb), expected) +class TestBranchRows(CustomTestCase): + """Two FFNs that branch from one input and merge again (LongCat's MoE and + dense branch): each layer's communicator gives the rows of its FFN input, + of its residual while the FFN runs and of what it hands on, and a complete + value moves between them.""" + + local = comm.Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER})) + attention = comm.Layout(frozenset({TokenAxis.ATTN_DP})) + full = comm.Layout(frozenset()) + + def rows(self, communicator, parallel, cp_extend=False): + batch = SimpleNamespace( + forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: cp_extend) + ) + with ( + planning(parallel), + patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), + patch.object( + comm, + "get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=False), + ), + patch.object(comm, "moe_cp_gathered_rows", lambda fb: [2, 1]), + ): + return communicator._branch_rows(batch) + + def test_each_branch_declares_its_rows(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + dense = build(layer_facts(2, 6), parallel) + self.assertEqual( + self.rows(dense, parallel), (self.full, self.attention, self.attention) + ) + for a2a, expected in ( + (True, (self.local, self.local, self.local)), + (False, (self.full, self.attention, self.attention)), + ): + with self.subTest(a2a=a2a): + moe = build( + layer_facts(1, 3, sparse=True, previous_sparse=True), + parallel, + a2a=a2a, + ) + self.assertEqual(self.rows(moe, parallel), expected) + + def test_a_batch_on_other_steps_has_no_branch_rows(self): + parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True) + communicator = build(layer_facts(1, 3), parallel) + with self.assertRaises(NotImplementedError): + self.rows(communicator, parallel, cp_extend=True) + + def test_branches_move_between_the_declared_rows(self): + # An a2a MoE beside a dense FFN, under attention DP and TP. + class Communicator(SimpleNamespace): + branch_input = LayerCommunicator.branch_input + branch_output = LayerCommunicator.branch_output + merge_branch = LayerCommunicator.merge_branch + + moe = Communicator(_branch_rows=lambda fb: (self.local, self.local, self.local)) + dense = Communicator( + _branch_rows=lambda fb: (self.full, self.attention, self.attention) + ) + moves = [] + + def recorded(value, rows, to, forward_batch): + moves.append((value, rows, to)) + return value + + with patch.object(comm, "move_rows", recorded): + dense.branch_input(moe, "h0", "residual", None) + self.assertEqual( + moves, + [ + ("h0", self.local, self.full), + ("residual", self.local, self.attention), + ], + ) + moves.clear() + moe.branch_output("shortcut", None) + self.assertEqual(moves, [("shortcut", self.local, self.local)]) + moves.clear() + merged = moe.merge_branch(1, 2, "residual", dense, None) + self.assertEqual( + moves, + [ + (2, self.attention, self.local), + ("residual", self.attention, self.local), + ], + ) + # The contribution adds; the residual is the dense branch's. + self.assertEqual(merged, (3, "residual")) + + def test_a_value_is_gathered_then_cut_in_order(self): + # Stand-ins that change the rows distinctly, so the order of the moves shows. + def tp_gather(h): + return torch.cat([h, h]) + + def dp_gather(h): + return torch.cat([h, h + 100]) + + def dp_scatter(h): + return h[: len(h) // 2] + + def tp_cut(h): + return h[len(h) // 2 :] + + value = torch.arange(4.0).view(4, 1) + with ( + patch.object(comm, "_redistribute_from_attn_tp_shards", tp_gather), + patch.object(comm, "_redistribute_input_to_dp", lambda h, fb: dp_gather(h)), + patch.object(comm, "_to_local_tokens", lambda step, fb, h: dp_scatter(h)), + patch.object( + comm, + "get_parallel", + lambda: SimpleNamespace(attn_tp_size=2, attn_tp_rank=1), + ), + ): + for rows, to, expected in ( + (self.local, self.full, dp_gather(tp_gather(value))), + (self.local, self.attention, tp_gather(value)), + (self.attention, self.local, tp_cut(value)), + (self.full, self.attention, dp_scatter(value)), + (self.full, self.local, tp_cut(dp_scatter(value))), + (self.attention, self.attention, value), + ): + with self.subTest(rows=rows, to=to): + self.assertTrue( + torch.equal(comm.move_rows(value, rows, to, None), expected) + ) + cp = comm.Layout(frozenset({TokenAxis.ATTN_CP})) + for rows, to in ( + (comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), self.attention), + (cp, self.full), + ): + with ( + self.subTest(rows=rows, to=to), + self.assertRaises(NotImplementedError), + ): + comm.move_rows(value, rows, to, None) + + # --------------------------------------------------------------------------- # Two consecutive layers, rank by rank. diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index e85c106247..207a2bf656 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -18,34 +18,29 @@ class TestLongcatShortcut(CustomTestCase): """On the last layer with scattered MoE tokens, the shortcut output must not add the residual a second time; the dense branch already carries it.""" tp, rows = 2, 4 - modes = comm.ScatterMode - context = comm.CommunicateContext( - process_group_sizes={ - modes.SCATTERED: 1, - modes.TP_ATTN_FULL: tp, - modes.FULL: tp, - }, - attn_tp_rank=0, - attn_tp_size=tp, - attn_dp_size=1, - attn_cp_rank=0, - attn_cp_size=1, - tp_size=tp, - tp_rank=0, - ) - postprocess = comm.CommunicateSummableTensorPairFn.get_fn( - modes.SCATTERED, modes.SCATTERED, modes.TP_ATTN_FULL, context - ) + # The MoE runs on each attention-TP rank's slice; the last layer hands on + # the attention's rows, where the dense branch ends. + local = comm.Layout(frozenset({comm.TokenAxis.ATTN_TP_SCATTER})) + attention = comm.Layout(frozenset()) + + class Communicator(SimpleNamespace): + branch_output = comm.LayerCommunicator.branch_output + merge_branch = comm.LayerCommunicator.merge_branch + fork_hidden = torch.full((rows // tp, 3), 2.0) fork_residual = torch.full_like(fork_hidden, 5.0) - preparation = SimpleNamespace( + moe_communicator = Communicator( prepare_attn=lambda h, r, batch: (h, r), prepare_mlp=lambda h, r, batch: (fork_hidden, fork_residual), - postprocess_layer=lambda h, r, batch: postprocess(h, r, batch, context), + _branch_rows=lambda batch: (local, local, attention), + ) + dense_communicator = Communicator( + _branch_rows=lambda batch: (attention, attention, attention) ) layer = LongcatFlashDecoderLayer.__new__(LongcatFlashDecoderLayer) nn.Module.__init__(layer) - layer.moe_layer_communicator = preparation + layer.moe_layer_communicator = moe_communicator + layer.mlp_layer_communicator = [None, dense_communicator] layer.self_attn = [lambda **kw: kw["hidden_states"]] layer.mlp = nn.Identity() layer.forward_mlp = Mock(