[Refactor] Support attention-TP dense FFNs and unpadded batches on stage boundaries (#42897)

This commit is contained in:
Cheng Wan
2026-10-09 01:27:00 -07:00
committed by GitHub
parent b9ae3d1a65
commit 2b469040d2
14 changed files with 200 additions and 27 deletions
@@ -26,6 +26,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.layout import (
batches_are_unpadded,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
@@ -159,9 +160,16 @@ def get_attn_tp_context():
def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
gathered = get_local_dp_buffer(
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
)
parallel = get_parallel()
if batches_are_unpadded():
# MLP sync, which sizes the local DP buffer for a batch, does not run.
gathered = tensor.new_empty(
(tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1])
)
else:
gathered = get_local_dp_buffer(
parallel.attn_tp_group, hidden_size=tensor.shape[-1]
)
attn_tp_all_gather_into_tensor(gathered, tensor)
return gathered
@@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import (
_batch_shards_over_cp,
_cp_gathers_over_attn_cp,
_prefill_cp_shards_tokens,
batches_are_unpadded,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.prepare import (
@@ -53,6 +54,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
is_moe_input_scattered_across_dp_ranks,
)
from sglang.srt.runtime_context import (
get_forward,
@@ -91,6 +93,20 @@ def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None
)
def _unpadded_possible() -> bool:
"""Whether a batch whose rows do not divide over attention TP may reach an
FFN that would run on this rank's attention-TP slice, so that FFN needs a
variant that stays on the attention's rows."""
return batches_are_unpadded() and (
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
)
def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool:
"""Whether this batch arrived with rows that do not divide over attention TP."""
return forward_batch.input_ids.shape[0] % attn_tp_size != 0
def _input_scattered_possible() -> bool:
"""Whether a batch may run this layer with input-scattered attention:
configured, on TP without attention DP, a prefill CP, an a2a backend or
@@ -173,6 +189,11 @@ class StagePlan:
):
self.norm = norm
self.edges = dict(variants)
# The attention TP size an unpadded batch's rows are checked against,
# when such a batch may arrive (see BatchVariant.UNPADDED).
self._unpadded_attn_tp_size = (
get_parallel().attn_tp_size if BatchVariant.UNPADDED in self.edges else None
)
self.enters_stack = enters_stack
self.terminal = terminal
self.finishes_directly = finishes_directly
@@ -240,6 +261,10 @@ class StagePlan:
return BatchVariant.INPUT_SCATTERED
if _batch_shards_over_cp(forward_batch):
return BatchVariant.CONTEXT_PARALLEL
if self._unpadded_attn_tp_size is not None and _rows_indivisible_over_attn_tp(
forward_batch, self._unpadded_attn_tp_size
):
return BatchVariant.UNPADDED
return BatchVariant.ORDINARY
def path_for(self, forward_batch):
@@ -52,6 +52,9 @@ class BatchVariant(Enum):
CONTEXT_PARALLEL = auto()
INPUT_SCATTERED = auto()
SEQUENCE_PARALLEL = auto()
# A batch whose rows do not divide over attention TP, which only arrives
# unpadded (--disable-attn-tp-gather without attention DP).
UNPADDED = auto()
class InputContract(msgspec.Struct, frozen=True):
@@ -22,6 +22,7 @@ from sglang.srt.layers.layer_boundary.construction import (
_bind_stage,
_input_scattered_possible,
_reject_unsupported_cp_moe,
_unpadded_possible,
_use_ag_after_qlora,
)
from sglang.srt.layers.layer_boundary.contracts import (
@@ -64,6 +65,8 @@ def _active_variants():
yield BatchVariant.INPUT_SCATTERED
if layernorm_sp.layernorm_sp_enabled():
yield BatchVariant.SEQUENCE_PARALLEL
if _unpadded_possible():
yield BatchVariant.UNPADDED
def _row_layouts(variant):
@@ -99,8 +102,11 @@ def _resolve_ffn(
output_complete=False,
):
parallel = get_parallel()
if dense_tp_size not in (None, 1, parallel.tp_size):
raise ValueError("dense FFN rows support local compute or the full TP group")
if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size):
raise ValueError(
"dense FFN rows support local compute, the attention TP group or the "
"full TP group"
)
strategy = get_exec().comm.boundary_reduction
if strategy not in ("ar", "rs", "rsv", "rs+rsv"):
raise ValueError(
@@ -114,13 +120,36 @@ def _resolve_ffn(
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
if parallel.attn_cp_size > 1 and sparse:
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
if variant is BatchVariant.UNPADDED and on_rank_rows:
# Rows that do not divide over attention TP stay whole: the FFN runs on
# the attention's rows (an a2a MoE dispatches them from every
# attention-TP rank) and its output is complete, as a2a's combine or
# local compute leaves it.
return (
StageContract(
InputContract(attention, read=read),
OutputContract(attention, update=update, transform=output_transform),
),
attention,
attention,
)
on_cp_shards = (
sparse
and parallel.attn_cp_size > 1
and parallel.moe_dp_size == parallel.attn_cp_size
and not _cp_gathers_over_attn_cp()
)
group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
# Over attention TP, short of the full TP group: the FFN owes only the
# attention-TP sum, and nothing moves over DP.
on_attention_rows = (
not sparse
and dense_tp_size == parallel.attn_tp_size
and dense_tp_size not in (1, parallel.tp_size)
)
if sparse:
group = SumGroup.MOE_OUTPUT
else:
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
if variant is BatchVariant.SEQUENCE_PARALLEL:
return (
StageContract(
@@ -160,13 +189,16 @@ def _resolve_ffn(
may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp_shards
else:
may_leave = may_scatter = True
rows = (
local
if on_rank_rows
else Layout.sharded_over(
if on_rank_rows:
rows = local
elif on_attention_rows:
rows = attention
# The output is already on the residual's rows: nothing to scatter.
use_reduce_scatter = use_reduce_scatterv = False
else:
rows = Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
)
)
# An FFN that writes the next stream itself hands on a complete output.
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
produced = (
@@ -219,7 +251,9 @@ class StageDeclaration:
tp_group: Group partitioning attention heads. Full TP consumers require
full token rows, including when TP shares the prefill CP group.
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
1 means local compute, the attention TP size means compute on the
attention's rows over attention TP, and the full TP size means TP
compute.
exit_rows: Required FFN output rows at the layer or branch exit.
writes_at_handoff: Whether this FFN writes its output into the residual
at its exit because the next stage is on another pipeline rank.
@@ -346,7 +380,7 @@ def declare_ffn(
next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only
to derive the TBO exit rows when exit_rows is not supplied.
dense_tp_size: Dense compute width: None for configuration, 1 for local
compute, or the full TP size.
compute, the attention TP size, or the full TP size.
exit_rows: Explicit output-row requirement; otherwise derived from
the adjacent FFN kinds and TBO configuration.
output_complete: Whether the compute completes its own output sum,
@@ -83,6 +83,18 @@ def is_dense_ffn_fully_dp():
return get_parallel().moe_dense_tp_size == 1
def batches_are_unpadded() -> bool:
"""Whether batches may reach the stages without padding to a multiple of
attention TP: --disable-attn-tp-gather skips that padding unless attention
DP is on."""
parallel = get_parallel()
return (
parallel.attn_tp_size > 1
and not parallel.attn_dp_enabled
and parallel.disable_attn_tp_gather
)
def _prefill_cp_shards_tokens() -> bool:
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
parallel = get_parallel()
+4 -11
View File
@@ -149,7 +149,6 @@ from sglang.srt.utils.common import (
add_prefix,
get_bool_env_var,
rank0_log,
require_mlp_sync,
set_weight_attrs,
)
@@ -2448,20 +2447,14 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
"""Whether the layers build stage boundaries, which is the same for every
layer of a stack. The layer's own communication still runs for: a dense
MLP sharded over attention TP; an attention-residual bank whose o_proj
all-reduce is fused with the pending add, or whose MoE on its attention-TP
token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded
carry also needs; and SP-MoE on batches that are not padded to a multiple
of attention TP (--disable-attn-tp-gather)."""
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled():
return False
layer of a stack. The layer's own communication still runs for: an
attention-residual bank whose o_proj all-reduce is fused with the pending
add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned
SP collectives, which the sharded carry also needs."""
if _fuses_attn_all_reduce(config):
return False
if not _shards_moe_rows():
return True
if not require_mlp_sync():
return False
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
+1
View File
@@ -95,6 +95,7 @@ def stub_plan():
plan.fusions = None
plan._publish_lora_layout = False
plan._next_input_rows = None
plan._unpadded_attn_tp_size = None
plan.paths = {}
plan.enters_stack = False
return plan