[Fix] LongCat-Flash under attention DP: branch and merge the dense FFNs through the communicators (#41436)
This commit is contained in:
@@ -1661,6 +1661,70 @@ class LayerCommunicator:
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch)
|
||||
|
||||
def _branch_rows(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Tuple[Layout, Layout, Layout]:
|
||||
"""The rows of this layer's FFN input, of its residual while the FFN
|
||||
runs, and of what the layer hands on, for a batch that runs its ordinary
|
||||
steps."""
|
||||
if self._batch_steps(forward_batch) is not self._steps:
|
||||
raise NotImplementedError(
|
||||
"a branch on a batch with attention CP, LayerNorm SP or "
|
||||
"input-scattered attention"
|
||||
)
|
||||
if self._declared is not None:
|
||||
sides = self._declared
|
||||
return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows
|
||||
layouts, modes = self._context.layouts, self.layer_scatter_modes
|
||||
return (
|
||||
layouts[modes.mlp_mode],
|
||||
layouts[modes.middle_residual_mode],
|
||||
layouts[modes.layer_output_mode],
|
||||
)
|
||||
|
||||
def branch_input(
|
||||
self,
|
||||
source: "LayerCommunicator",
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The FFN input and residual that ``source``'s boundary read for its
|
||||
own FFN, for this layer's FFN, which branches from the same input: moved
|
||||
to the rows this FFN needs and its residual's rows."""
|
||||
rows, residual_rows, _ = source._branch_rows(forward_batch)
|
||||
to, residual_to, _ = self._branch_rows(forward_batch)
|
||||
return (
|
||||
move_rows(hidden_states, rows, to, forward_batch),
|
||||
move_rows(residual, residual_rows, residual_to, forward_batch),
|
||||
)
|
||||
|
||||
def branch_output(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
"""This layer's complete FFN output as a branch's contribution, which
|
||||
adds to the layer's output without writing the residual: moved to the
|
||||
rows the layer hands on."""
|
||||
rows, _, to = self._branch_rows(forward_batch)
|
||||
return move_rows(hidden_states, rows, to, forward_batch)
|
||||
|
||||
def merge_branch(
|
||||
self,
|
||||
contribution: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
source: "LayerCommunicator",
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A contribution from ``branch_output`` summed with what ``source``'s
|
||||
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
|
||||
this layer hands on."""
|
||||
_, _, rows = source._branch_rows(forward_batch)
|
||||
_, _, to = self._branch_rows(forward_batch)
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, rows, to, forward_batch)
|
||||
return contribution + hidden_states, residual
|
||||
|
||||
def finish_layer_stack(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
@@ -2139,6 +2203,34 @@ def _reduce_and_redistribute_output_to_dp(
|
||||
return global_hidden_states
|
||||
|
||||
|
||||
def move_rows(
|
||||
hidden_states: torch.Tensor,
|
||||
rows: Layout,
|
||||
to: Layout,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
"""A complete value from the rows it is on to ``to``: gathered over the
|
||||
token axes ``to`` does not shard (attention TP, then attention DP), or cut to
|
||||
this rank's share of those it does (attention DP, then attention TP)."""
|
||||
gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded
|
||||
if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut:
|
||||
raise NotImplementedError(f"{rows=} {to=}")
|
||||
if TokenAxis.ATTN_TP_SCATTER in gathered:
|
||||
hidden_states = _redistribute_from_attn_tp_shards(hidden_states)
|
||||
if TokenAxis.ATTN_DP in gathered:
|
||||
hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch)
|
||||
if TokenAxis.ATTN_DP in cut:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
if TokenAxis.ATTN_TP_SCATTER in cut:
|
||||
parallel = get_parallel()
|
||||
hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
|
||||
parallel.attn_tp_rank
|
||||
]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
"""A kernel that completes the sum the attention output owes together with
|
||||
the residual add and the post-attention norm, in that order.
|
||||
|
||||
@@ -90,9 +90,8 @@ from sglang.srt.model_loader.utils import (
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel as _gp
|
||||
from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_stream,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
@@ -140,23 +139,6 @@ _LONGCAT_FLASH_ROUTER_HPC_GEMM_MIN_M = {
|
||||
}
|
||||
|
||||
|
||||
def _scmoe_align_rows(t, target):
|
||||
"""Align a [rows,H] tensor to `target` rows across the attn-tp group:
|
||||
all_gather when target>rows (target==rows*attn_tp_size), or take this rank's
|
||||
contiguous segment when target<rows."""
|
||||
if t is None or t.shape[0] == target:
|
||||
return t
|
||||
from sglang.srt.layers.dp_attention import attn_tp_all_gather_into_tensor as _ag
|
||||
|
||||
cur = t.shape[0]
|
||||
if target > cur:
|
||||
out = t.new_empty((target, *t.shape[1:]))
|
||||
_ag(out, t.contiguous())
|
||||
return out
|
||||
r = _gp().attn_tp_rank
|
||||
return t[r * target : r * target + target].contiguous()
|
||||
|
||||
|
||||
class LongcatFlashMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -420,6 +402,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
if "mlps" in getattr(config, "disable_quant_module", [])
|
||||
else quant_config
|
||||
),
|
||||
reduce_results=True,
|
||||
prefix=add_prefix(f"mlps.{i}", prefix),
|
||||
)
|
||||
for i in range(2)
|
||||
@@ -453,6 +436,7 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm[i],
|
||||
post_attention_layernorm=self.post_attention_layernorm[i],
|
||||
qkv_latent_func=self.self_attn[i].prepare_qkv_latent,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
@@ -502,12 +486,8 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.moe_layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
moe_hidden_states = hidden_states.clone()
|
||||
# The final gather adds its residual; the dense branch already carries it.
|
||||
moe_residual = torch.zeros_like(residual)
|
||||
moe_hidden_states = self.mlp(moe_hidden_states)
|
||||
moe_hidden_states, moe_residual = self.moe_layer_communicator.postprocess_layer(
|
||||
moe_hidden_states, moe_residual, forward_batch
|
||||
moe_hidden_states = self.moe_layer_communicator.branch_output(
|
||||
self.mlp(hidden_states.clone()), forward_batch
|
||||
)
|
||||
|
||||
hidden_states, residual, prev_topk_indices = self.forward_mlp(
|
||||
@@ -519,12 +499,13 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prev_topk_indices,
|
||||
)
|
||||
|
||||
# SCMOE_ATTN1_GATHER: reconcile mlp-branch (hidden/residual) with moe-branch
|
||||
# (moe_hidden_states) row counts before the add, so residual tracks hidden.
|
||||
_scmoe_tgt = moe_hidden_states.shape[0]
|
||||
hidden_states = _scmoe_align_rows(hidden_states, _scmoe_tgt)
|
||||
residual = _scmoe_align_rows(residual, _scmoe_tgt)
|
||||
hidden_states = moe_hidden_states + hidden_states
|
||||
hidden_states, residual = self.moe_layer_communicator.merge_branch(
|
||||
moe_hidden_states,
|
||||
hidden_states,
|
||||
residual,
|
||||
self.mlp_layer_communicator[1],
|
||||
forward_batch,
|
||||
)
|
||||
return hidden_states, residual, prev_topk_indices
|
||||
|
||||
def forward_mlp(
|
||||
@@ -536,25 +517,13 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
zero_allocator,
|
||||
prev_topk_indices,
|
||||
):
|
||||
# SCMOE_ATTN1_GATHER: gather the dense-branch hidden(+residual) to full
|
||||
# tokens before mlps[0] so its all_reduce is valid; slice back at the merge.
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend as _scmoe_gab
|
||||
|
||||
_scmoe_ats = self.attn_tp_size
|
||||
_scmoe_do = (
|
||||
not _scmoe_gab().is_none()
|
||||
and _scmoe_ats > 1
|
||||
and hidden_states.shape[0] != 0
|
||||
and hidden_states.shape[0] * _scmoe_ats == positions.shape[0]
|
||||
# first_mlp, on the input the MoE's boundary read
|
||||
hidden_states, residual = self.mlp_layer_communicator[0].branch_input(
|
||||
self.moe_layer_communicator, hidden_states, residual, forward_batch
|
||||
)
|
||||
if _scmoe_do:
|
||||
hidden_states = _scmoe_align_rows(hidden_states, positions.shape[0])
|
||||
residual = _scmoe_align_rows(residual, positions.shape[0])
|
||||
|
||||
# first_mlp
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
# TP all_reduce
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
with self.mlp_layer_communicator[0].ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[0](hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
# second_attn
|
||||
hidden_states, residual = self.mlp_layer_communicator[1].prepare_attn(
|
||||
@@ -577,13 +546,9 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.mlp_layer_communicator[1].prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
# TP all_reduce
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
hidden_states, residual = self.mlp_layer_communicator[1].postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.mlp_layer_communicator[1].ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlps[1](hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual, prev_topk_indices
|
||||
|
||||
|
||||
Reference in New Issue
Block a user