[Fix] LongCat-Flash under attention DP: branch and merge the dense FFNs through the communicators (#41436)

This commit is contained in:
Cheng Wan
2026-09-27 16:53:36 -07:00
committed by GitHub
parent 4fac02fe5d
commit 0971450c16
5 changed files with 298 additions and 83 deletions
+92
View File
@@ -1661,6 +1661,70 @@ class LayerCommunicator:
result as a context manager around the FFN call, then call ``finish``."""
return FfnExit(self, forward_batch)
def _branch_rows(
self, forward_batch: ForwardBatch
) -> Tuple[Layout, Layout, Layout]:
"""The rows of this layer's FFN input, of its residual while the FFN
runs, and of what the layer hands on, for a batch that runs its ordinary
steps."""
if self._batch_steps(forward_batch) is not self._steps:
raise NotImplementedError(
"a branch on a batch with attention CP, LayerNorm SP or "
"input-scattered attention"
)
if self._declared is not None:
sides = self._declared
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
layouts, modes = self._context.layouts, self.layer_scatter_modes
return (
layouts[modes.mlp_mode],
layouts[modes.middle_residual_mode],
layouts[modes.layer_output_mode],
)
def branch_input(
self,
source: "LayerCommunicator",
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""The FFN input and residual that ``source``'s boundary read for its
own FFN, for this layer's FFN, which branches from the same input: moved
to the rows this FFN needs and its residual's rows."""
rows, residual_rows, _ = source._branch_rows(forward_batch)
to, residual_to, _ = self._branch_rows(forward_batch)
return (
move_rows(hidden_states, rows, to, forward_batch),
move_rows(residual, residual_rows, residual_to, forward_batch),
)
def branch_output(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
"""This layer's complete FFN output as a branch's contribution, which
adds to the layer's output without writing the residual: moved to the
rows the layer hands on."""
rows, _, to = self._branch_rows(forward_batch)
return move_rows(hidden_states, rows, to, forward_batch)
def merge_branch(
self,
contribution: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor,
source: "LayerCommunicator",
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""A contribution from ``branch_output`` summed with what ``source``'s
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
this layer hands on."""
_, _, rows = source._branch_rows(forward_batch)
_, _, to = self._branch_rows(forward_batch)
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, rows, to, forward_batch)
return contribution + hidden_states, residual
def finish_layer_stack(
self,
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
@@ -2139,6 +2203,34 @@ def _reduce_and_redistribute_output_to_dp(
return global_hidden_states
def move_rows(
hidden_states: torch.Tensor,
rows: Layout,
to: Layout,
forward_batch: ForwardBatch,
) -> torch.Tensor:
"""A complete value from the rows it is on to ``to``: gathered over the
token axes ``to`` does not shard (attention TP, then attention DP), or cut to
this rank's share of those it does (attention DP, then attention TP)."""
gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded
if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut:
raise NotImplementedError(f"{rows=} {to=}")
if TokenAxis.ATTN_TP_SCATTER in gathered:
hidden_states = _redistribute_from_attn_tp_shards(hidden_states)
if TokenAxis.ATTN_DP in gathered:
hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch)
if TokenAxis.ATTN_DP in cut:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
if TokenAxis.ATTN_TP_SCATTER in cut:
parallel = get_parallel()
hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
parallel.attn_tp_rank
]
return hidden_states
class FusedMlpInput(msgspec.Struct, frozen=True):
"""A kernel that completes the sum the attention output owes together with
the residual add and the post-attention norm, in that order.
+21 -56
View File
@@ -90,9 +90,8 @@ from sglang.srt.model_loader.utils import (
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_parallel as _gp
from sglang.srt.runtime_context import (
get_parallel,
get_stream,
)
from sglang.srt.utils import (
@@ -140,23 +139,6 @@ _LONGCAT_FLASH_ROUTER_HPC_GEMM_MIN_M = {
}
def _scmoe_align_rows(t, target):
"""Align a [rows,H] tensor to `target` rows across the attn-tp group:
all_gather when target>rows (target==rows*attn_tp_size), or take this rank's
contiguous segment when target<rows."""
if t is None or t.shape[0] == target:
return t
from sglang.srt.layers.dp_attention import attn_tp_all_gather_into_tensor as _ag
cur = t.shape[0]
if target > cur:
out = t.new_empty((target, *t.shape[1:]))
_ag(out, t.contiguous())
return out
r = _gp().attn_tp_rank
return t[r * target : r * target + target].contiguous()
class LongcatFlashMLP(nn.Module):
def __init__(
self,
@@ -420,6 +402,7 @@ class LongcatFlashDecoderLayer(nn.Module):
if "mlps" in getattr(config, "disable_quant_module", [])
else quant_config
),
reduce_results=True,
prefix=add_prefix(f"mlps.{i}", prefix),
)
for i in range(2)
@@ -453,6 +436,7 @@ class LongcatFlashDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm[i],
post_attention_layernorm=self.post_attention_layernorm[i],
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
allow_deferred_ffn_reduction=False,
)
for i in range(2)
]
@@ -502,12 +486,8 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states, residual = self.moe_layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
moe_hidden_states = hidden_states.clone()
# The final gather adds its residual; the dense branch already carries it.
moe_residual = torch.zeros_like(residual)
moe_hidden_states = self.mlp(moe_hidden_states)
moe_hidden_states, moe_residual = self.moe_layer_communicator.postprocess_layer(
moe_hidden_states, moe_residual, forward_batch
moe_hidden_states = self.moe_layer_communicator.branch_output(
self.mlp(hidden_states.clone()), forward_batch
)
hidden_states, residual, prev_topk_indices = self.forward_mlp(
@@ -519,12 +499,13 @@ class LongcatFlashDecoderLayer(nn.Module):
prev_topk_indices,
)
# SCMOE_ATTN1_GATHER: reconcile mlp-branch (hidden/residual) with moe-branch
# (moe_hidden_states) row counts before the add, so residual tracks hidden.
_scmoe_tgt = moe_hidden_states.shape[0]
hidden_states = _scmoe_align_rows(hidden_states, _scmoe_tgt)
residual = _scmoe_align_rows(residual, _scmoe_tgt)
hidden_states = moe_hidden_states + hidden_states
hidden_states, residual = self.moe_layer_communicator.merge_branch(
moe_hidden_states,
hidden_states,
residual,
self.mlp_layer_communicator[1],
forward_batch,
)
return hidden_states, residual, prev_topk_indices
def forward_mlp(
@@ -536,25 +517,13 @@ class LongcatFlashDecoderLayer(nn.Module):
zero_allocator,
prev_topk_indices,
):
# SCMOE_ATTN1_GATHER: gather the dense-branch hidden(+residual) to full
# tokens before mlps[0] so its all_reduce is valid; slice back at the merge.
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _scmoe_gab
_scmoe_ats = self.attn_tp_size
_scmoe_do = (
not _scmoe_gab().is_none()
and _scmoe_ats > 1
and hidden_states.shape[0] != 0
and hidden_states.shape[0] * _scmoe_ats == positions.shape[0]
# first_mlp, on the input the MoE's boundary read
hidden_states, residual = self.mlp_layer_communicator[0].branch_input(
self.moe_layer_communicator, hidden_states, residual, forward_batch
)
if _scmoe_do:
hidden_states = _scmoe_align_rows(hidden_states, positions.shape[0])
residual = _scmoe_align_rows(residual, positions.shape[0])
# first_mlp
hidden_states = self.mlps[0](hidden_states)
# TP all_reduce
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[0](hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
# second_attn
hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn(
@@ -577,13 +546,9 @@ class LongcatFlashDecoderLayer(nn.Module):
hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.mlps[1](hidden_states)
# TP all_reduce
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states, residual = self.mlp_layer_communicator[1].postprocess_layer(
hidden_states, residual, forward_batch
)
with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlps[1](hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual, prev_topk_indices