[Refactor] Support attention-TP dense FFNs and unpadded batches on stage boundaries (#42897)
This commit is contained in:
@@ -26,6 +26,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
@@ -159,9 +160,16 @@ def get_attn_tp_context():
|
||||
|
||||
|
||||
def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
|
||||
gathered = get_local_dp_buffer(
|
||||
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
parallel = get_parallel()
|
||||
if batches_are_unpadded():
|
||||
# MLP sync, which sizes the local DP buffer for a batch, does not run.
|
||||
gathered = tensor.new_empty(
|
||||
(tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1])
|
||||
)
|
||||
else:
|
||||
gathered = get_local_dp_buffer(
|
||||
parallel.attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(gathered, tensor)
|
||||
return gathered
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
_batch_shards_over_cp,
|
||||
_cp_gathers_over_attn_cp,
|
||||
_prefill_cp_shards_tokens,
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
@@ -53,6 +54,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
@@ -91,6 +93,20 @@ def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None
|
||||
)
|
||||
|
||||
|
||||
def _unpadded_possible() -> bool:
|
||||
"""Whether a batch whose rows do not divide over attention TP may reach an
|
||||
FFN that would run on this rank's attention-TP slice, so that FFN needs a
|
||||
variant that stays on the attention's rows."""
|
||||
return batches_are_unpadded() and (
|
||||
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool:
|
||||
"""Whether this batch arrived with rows that do not divide over attention TP."""
|
||||
return forward_batch.input_ids.shape[0] % attn_tp_size != 0
|
||||
|
||||
|
||||
def _input_scattered_possible() -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on TP without attention DP, a prefill CP, an a2a backend or
|
||||
@@ -173,6 +189,11 @@ class StagePlan:
|
||||
):
|
||||
self.norm = norm
|
||||
self.edges = dict(variants)
|
||||
# The attention TP size an unpadded batch's rows are checked against,
|
||||
# when such a batch may arrive (see BatchVariant.UNPADDED).
|
||||
self._unpadded_attn_tp_size = (
|
||||
get_parallel().attn_tp_size if BatchVariant.UNPADDED in self.edges else None
|
||||
)
|
||||
self.enters_stack = enters_stack
|
||||
self.terminal = terminal
|
||||
self.finishes_directly = finishes_directly
|
||||
@@ -240,6 +261,10 @@ class StagePlan:
|
||||
return BatchVariant.INPUT_SCATTERED
|
||||
if _batch_shards_over_cp(forward_batch):
|
||||
return BatchVariant.CONTEXT_PARALLEL
|
||||
if self._unpadded_attn_tp_size is not None and _rows_indivisible_over_attn_tp(
|
||||
forward_batch, self._unpadded_attn_tp_size
|
||||
):
|
||||
return BatchVariant.UNPADDED
|
||||
return BatchVariant.ORDINARY
|
||||
|
||||
def path_for(self, forward_batch):
|
||||
|
||||
@@ -52,6 +52,9 @@ class BatchVariant(Enum):
|
||||
CONTEXT_PARALLEL = auto()
|
||||
INPUT_SCATTERED = auto()
|
||||
SEQUENCE_PARALLEL = auto()
|
||||
# A batch whose rows do not divide over attention TP, which only arrives
|
||||
# unpadded (--disable-attn-tp-gather without attention DP).
|
||||
UNPADDED = auto()
|
||||
|
||||
|
||||
class InputContract(msgspec.Struct, frozen=True):
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.layers.layer_boundary.construction import (
|
||||
_bind_stage,
|
||||
_input_scattered_possible,
|
||||
_reject_unsupported_cp_moe,
|
||||
_unpadded_possible,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
@@ -64,6 +65,8 @@ def _active_variants():
|
||||
yield BatchVariant.INPUT_SCATTERED
|
||||
if layernorm_sp.layernorm_sp_enabled():
|
||||
yield BatchVariant.SEQUENCE_PARALLEL
|
||||
if _unpadded_possible():
|
||||
yield BatchVariant.UNPADDED
|
||||
|
||||
|
||||
def _row_layouts(variant):
|
||||
@@ -99,8 +102,11 @@ def _resolve_ffn(
|
||||
output_complete=False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if dense_tp_size not in (None, 1, parallel.tp_size):
|
||||
raise ValueError("dense FFN rows support local compute or the full TP group")
|
||||
if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size):
|
||||
raise ValueError(
|
||||
"dense FFN rows support local compute, the attention TP group or the "
|
||||
"full TP group"
|
||||
)
|
||||
strategy = get_exec().comm.boundary_reduction
|
||||
if strategy not in ("ar", "rs", "rsv", "rs+rsv"):
|
||||
raise ValueError(
|
||||
@@ -114,13 +120,36 @@ def _resolve_ffn(
|
||||
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
|
||||
if variant is BatchVariant.UNPADDED and on_rank_rows:
|
||||
# Rows that do not divide over attention TP stay whole: the FFN runs on
|
||||
# the attention's rows (an a2a MoE dispatches them from every
|
||||
# attention-TP rank) and its output is complete, as a2a's combine or
|
||||
# local compute leaves it.
|
||||
return (
|
||||
StageContract(
|
||||
InputContract(attention, read=read),
|
||||
OutputContract(attention, update=update, transform=output_transform),
|
||||
),
|
||||
attention,
|
||||
attention,
|
||||
)
|
||||
on_cp_shards = (
|
||||
sparse
|
||||
and parallel.attn_cp_size > 1
|
||||
and parallel.moe_dp_size == parallel.attn_cp_size
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
)
|
||||
group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
|
||||
# Over attention TP, short of the full TP group: the FFN owes only the
|
||||
# attention-TP sum, and nothing moves over DP.
|
||||
on_attention_rows = (
|
||||
not sparse
|
||||
and dense_tp_size == parallel.attn_tp_size
|
||||
and dense_tp_size not in (1, parallel.tp_size)
|
||||
)
|
||||
if sparse:
|
||||
group = SumGroup.MOE_OUTPUT
|
||||
else:
|
||||
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
|
||||
if variant is BatchVariant.SEQUENCE_PARALLEL:
|
||||
return (
|
||||
StageContract(
|
||||
@@ -160,13 +189,16 @@ def _resolve_ffn(
|
||||
may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp_shards
|
||||
else:
|
||||
may_leave = may_scatter = True
|
||||
rows = (
|
||||
local
|
||||
if on_rank_rows
|
||||
else Layout.sharded_over(
|
||||
if on_rank_rows:
|
||||
rows = local
|
||||
elif on_attention_rows:
|
||||
rows = attention
|
||||
# The output is already on the residual's rows: nothing to scatter.
|
||||
use_reduce_scatter = use_reduce_scatterv = False
|
||||
else:
|
||||
rows = Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
|
||||
)
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
|
||||
produced = (
|
||||
@@ -219,7 +251,9 @@ class StageDeclaration:
|
||||
tp_group: Group partitioning attention heads. Full TP consumers require
|
||||
full token rows, including when TP shares the prefill CP group.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
1 means local compute, the attention TP size means compute on the
|
||||
attention's rows over attention TP, and the full TP size means TP
|
||||
compute.
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
writes_at_handoff: Whether this FFN writes its output into the residual
|
||||
at its exit because the next stage is on another pipeline rank.
|
||||
@@ -346,7 +380,7 @@ def declare_ffn(
|
||||
next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only
|
||||
to derive the TBO exit rows when exit_rows is not supplied.
|
||||
dense_tp_size: Dense compute width: None for configuration, 1 for local
|
||||
compute, or the full TP size.
|
||||
compute, the attention TP size, or the full TP size.
|
||||
exit_rows: Explicit output-row requirement; otherwise derived from
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
|
||||
@@ -83,6 +83,18 @@ def is_dense_ffn_fully_dp():
|
||||
return get_parallel().moe_dense_tp_size == 1
|
||||
|
||||
|
||||
def batches_are_unpadded() -> bool:
|
||||
"""Whether batches may reach the stages without padding to a multiple of
|
||||
attention TP: --disable-attn-tp-gather skips that padding unless attention
|
||||
DP is on."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.attn_tp_size > 1
|
||||
and not parallel.attn_dp_enabled
|
||||
and parallel.disable_attn_tp_gather
|
||||
)
|
||||
|
||||
|
||||
def _prefill_cp_shards_tokens() -> bool:
|
||||
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
|
||||
parallel = get_parallel()
|
||||
|
||||
@@ -149,7 +149,6 @@ from sglang.srt.utils.common import (
|
||||
add_prefix,
|
||||
get_bool_env_var,
|
||||
rank0_log,
|
||||
require_mlp_sync,
|
||||
set_weight_attrs,
|
||||
)
|
||||
|
||||
@@ -2448,20 +2447,14 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
|
||||
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the layers build stage boundaries, which is the same for every
|
||||
layer of a stack. The layer's own communication still runs for: a dense
|
||||
MLP sharded over attention TP; an attention-residual bank whose o_proj
|
||||
all-reduce is fused with the pending add, or whose MoE on its attention-TP
|
||||
token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded
|
||||
carry also needs; and SP-MoE on batches that are not padded to a multiple
|
||||
of attention TP (--disable-attn-tp-gather)."""
|
||||
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled():
|
||||
return False
|
||||
layer of a stack. The layer's own communication still runs for: an
|
||||
attention-residual bank whose o_proj all-reduce is fused with the pending
|
||||
add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned
|
||||
SP collectives, which the sharded carry also needs."""
|
||||
if _fuses_attn_all_reduce(config):
|
||||
return False
|
||||
if not _shards_moe_rows():
|
||||
return True
|
||||
if not require_mlp_sync():
|
||||
return False
|
||||
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ def stub_plan():
|
||||
plan.fusions = None
|
||||
plan._publish_lora_layout = False
|
||||
plan._next_input_rows = None
|
||||
plan._unpadded_attn_tp_size = None
|
||||
plan.paths = {}
|
||||
plan.enters_stack = False
|
||||
return plan
|
||||
|
||||
Reference in New Issue
Block a user