[Refactor] Support attention-TP dense FFNs and unpadded batches on stage boundaries (#42897)
This commit is contained in:
@@ -262,7 +262,7 @@ Common errors and their usual cause:
|
||||
| `start the layer stack before entering a stage` | The forward path, a split-prefill segment, or an MTP head skipped `residual_batch.start()` |
|
||||
| `output does not belong to this residual stream; change layer outputs through boundary accessors` | The model replaced a layer's output object (for example rebinding it to a new tensor) or passed another microbatch's output. This check compares object identity, so it does not catch in-place changes; the interface still forbids modifying layer outputs in place |
|
||||
| `write the residual update before taking the final output` | `take_output()` ran while a contribution was still pending; fold or norm it first |
|
||||
| `no stage boundary path for the active <VARIANT> batch` | The active batch variant (CP, input-scattered, sequence parallel) is not supported by this stage; add a server-argument check that rejects the configuration |
|
||||
| `no stage boundary path for the active <VARIANT> batch` | The active batch variant (CP, input-scattered, sequence parallel, unpadded) is not supported by this stage; add a server-argument check that rejects the configuration |
|
||||
| `append_stages needs an open layer stack` | The layers are built outside `make_layers()`; wrap their construction in `layer_stack()` |
|
||||
| `a prepared branch must enter through branch_input, not prepare` | A `prepared_from` stage was entered with `prepare()` |
|
||||
| `snapshot requires a plain residual update` | A snapshot was taken on a nonlinear (for example MHC) update |
|
||||
@@ -323,7 +323,7 @@ Models normally use factories, `StageBoundary`, and `residual.batch`. Keep row a
|
||||
|
||||
### Batch variants
|
||||
|
||||
`StagePlan` preconstructs ordinary, CP, input-scattered, and sequence-parallel variants when supported by the configured topology. A forward selects a path from active runtime facts. An absent active variant raises an error rather than silently using ordinary token rows.
|
||||
`StagePlan` preconstructs ordinary, CP, input-scattered, sequence-parallel, and unpadded variants when supported by the configured topology. The unpadded variant serves batches whose rows do not divide over attention TP, which `--disable-attn-tp-gather` lets through without attention DP: an FFN that would run on this rank's attention-TP slice stays on the attention's rows instead. A forward selects a path from active runtime facts. An absent active variant raises an error rather than silently using ordinary token rows.
|
||||
|
||||
Configuration validation rejects unsupported model/parallel combinations. Boundary construction also rejects transitions without a correct implementation. Adding an enum value or a layout does not by itself provide a new collective path.
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
@@ -159,9 +160,16 @@ def get_attn_tp_context():
|
||||
|
||||
|
||||
def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
|
||||
gathered = get_local_dp_buffer(
|
||||
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
parallel = get_parallel()
|
||||
if batches_are_unpadded():
|
||||
# MLP sync, which sizes the local DP buffer for a batch, does not run.
|
||||
gathered = tensor.new_empty(
|
||||
(tensor.shape[0] * parallel.attn_tp_size, tensor.shape[-1])
|
||||
)
|
||||
else:
|
||||
gathered = get_local_dp_buffer(
|
||||
parallel.attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(gathered, tensor)
|
||||
return gathered
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
_batch_shards_over_cp,
|
||||
_cp_gathers_over_attn_cp,
|
||||
_prefill_cp_shards_tokens,
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
@@ -53,6 +54,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
@@ -91,6 +93,20 @@ def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None
|
||||
)
|
||||
|
||||
|
||||
def _unpadded_possible() -> bool:
|
||||
"""Whether a batch whose rows do not divide over attention TP may reach an
|
||||
FFN that would run on this rank's attention-TP slice, so that FFN needs a
|
||||
variant that stays on the attention's rows."""
|
||||
return batches_are_unpadded() and (
|
||||
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool:
|
||||
"""Whether this batch arrived with rows that do not divide over attention TP."""
|
||||
return forward_batch.input_ids.shape[0] % attn_tp_size != 0
|
||||
|
||||
|
||||
def _input_scattered_possible() -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on TP without attention DP, a prefill CP, an a2a backend or
|
||||
@@ -173,6 +189,11 @@ class StagePlan:
|
||||
):
|
||||
self.norm = norm
|
||||
self.edges = dict(variants)
|
||||
# The attention TP size an unpadded batch's rows are checked against,
|
||||
# when such a batch may arrive (see BatchVariant.UNPADDED).
|
||||
self._unpadded_attn_tp_size = (
|
||||
get_parallel().attn_tp_size if BatchVariant.UNPADDED in self.edges else None
|
||||
)
|
||||
self.enters_stack = enters_stack
|
||||
self.terminal = terminal
|
||||
self.finishes_directly = finishes_directly
|
||||
@@ -240,6 +261,10 @@ class StagePlan:
|
||||
return BatchVariant.INPUT_SCATTERED
|
||||
if _batch_shards_over_cp(forward_batch):
|
||||
return BatchVariant.CONTEXT_PARALLEL
|
||||
if self._unpadded_attn_tp_size is not None and _rows_indivisible_over_attn_tp(
|
||||
forward_batch, self._unpadded_attn_tp_size
|
||||
):
|
||||
return BatchVariant.UNPADDED
|
||||
return BatchVariant.ORDINARY
|
||||
|
||||
def path_for(self, forward_batch):
|
||||
|
||||
@@ -52,6 +52,9 @@ class BatchVariant(Enum):
|
||||
CONTEXT_PARALLEL = auto()
|
||||
INPUT_SCATTERED = auto()
|
||||
SEQUENCE_PARALLEL = auto()
|
||||
# A batch whose rows do not divide over attention TP, which only arrives
|
||||
# unpadded (--disable-attn-tp-gather without attention DP).
|
||||
UNPADDED = auto()
|
||||
|
||||
|
||||
class InputContract(msgspec.Struct, frozen=True):
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.layers.layer_boundary.construction import (
|
||||
_bind_stage,
|
||||
_input_scattered_possible,
|
||||
_reject_unsupported_cp_moe,
|
||||
_unpadded_possible,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
@@ -64,6 +65,8 @@ def _active_variants():
|
||||
yield BatchVariant.INPUT_SCATTERED
|
||||
if layernorm_sp.layernorm_sp_enabled():
|
||||
yield BatchVariant.SEQUENCE_PARALLEL
|
||||
if _unpadded_possible():
|
||||
yield BatchVariant.UNPADDED
|
||||
|
||||
|
||||
def _row_layouts(variant):
|
||||
@@ -99,8 +102,11 @@ def _resolve_ffn(
|
||||
output_complete=False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if dense_tp_size not in (None, 1, parallel.tp_size):
|
||||
raise ValueError("dense FFN rows support local compute or the full TP group")
|
||||
if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size):
|
||||
raise ValueError(
|
||||
"dense FFN rows support local compute, the attention TP group or the "
|
||||
"full TP group"
|
||||
)
|
||||
strategy = get_exec().comm.boundary_reduction
|
||||
if strategy not in ("ar", "rs", "rsv", "rs+rsv"):
|
||||
raise ValueError(
|
||||
@@ -114,13 +120,36 @@ def _resolve_ffn(
|
||||
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
|
||||
if variant is BatchVariant.UNPADDED and on_rank_rows:
|
||||
# Rows that do not divide over attention TP stay whole: the FFN runs on
|
||||
# the attention's rows (an a2a MoE dispatches them from every
|
||||
# attention-TP rank) and its output is complete, as a2a's combine or
|
||||
# local compute leaves it.
|
||||
return (
|
||||
StageContract(
|
||||
InputContract(attention, read=read),
|
||||
OutputContract(attention, update=update, transform=output_transform),
|
||||
),
|
||||
attention,
|
||||
attention,
|
||||
)
|
||||
on_cp_shards = (
|
||||
sparse
|
||||
and parallel.attn_cp_size > 1
|
||||
and parallel.moe_dp_size == parallel.attn_cp_size
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
)
|
||||
group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
|
||||
# Over attention TP, short of the full TP group: the FFN owes only the
|
||||
# attention-TP sum, and nothing moves over DP.
|
||||
on_attention_rows = (
|
||||
not sparse
|
||||
and dense_tp_size == parallel.attn_tp_size
|
||||
and dense_tp_size not in (1, parallel.tp_size)
|
||||
)
|
||||
if sparse:
|
||||
group = SumGroup.MOE_OUTPUT
|
||||
else:
|
||||
group = SumGroup.ATTN_TP if on_attention_rows else SumGroup.TP
|
||||
if variant is BatchVariant.SEQUENCE_PARALLEL:
|
||||
return (
|
||||
StageContract(
|
||||
@@ -160,13 +189,16 @@ def _resolve_ffn(
|
||||
may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp_shards
|
||||
else:
|
||||
may_leave = may_scatter = True
|
||||
rows = (
|
||||
local
|
||||
if on_rank_rows
|
||||
else Layout.sharded_over(
|
||||
if on_rank_rows:
|
||||
rows = local
|
||||
elif on_attention_rows:
|
||||
rows = attention
|
||||
# The output is already on the residual's rows: nothing to scatter.
|
||||
use_reduce_scatter = use_reduce_scatterv = False
|
||||
else:
|
||||
rows = Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp_shards else ()), axis_sizes=axes
|
||||
)
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
|
||||
produced = (
|
||||
@@ -219,7 +251,9 @@ class StageDeclaration:
|
||||
tp_group: Group partitioning attention heads. Full TP consumers require
|
||||
full token rows, including when TP shares the prefill CP group.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
1 means local compute, the attention TP size means compute on the
|
||||
attention's rows over attention TP, and the full TP size means TP
|
||||
compute.
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
writes_at_handoff: Whether this FFN writes its output into the residual
|
||||
at its exit because the next stage is on another pipeline rank.
|
||||
@@ -346,7 +380,7 @@ def declare_ffn(
|
||||
next_layer_sparse: Whether the next decoder layer's FFN is sparse; used only
|
||||
to derive the TBO exit rows when exit_rows is not supplied.
|
||||
dense_tp_size: Dense compute width: None for configuration, 1 for local
|
||||
compute, or the full TP size.
|
||||
compute, the attention TP size, or the full TP size.
|
||||
exit_rows: Explicit output-row requirement; otherwise derived from
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
|
||||
@@ -83,6 +83,18 @@ def is_dense_ffn_fully_dp():
|
||||
return get_parallel().moe_dense_tp_size == 1
|
||||
|
||||
|
||||
def batches_are_unpadded() -> bool:
|
||||
"""Whether batches may reach the stages without padding to a multiple of
|
||||
attention TP: --disable-attn-tp-gather skips that padding unless attention
|
||||
DP is on."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.attn_tp_size > 1
|
||||
and not parallel.attn_dp_enabled
|
||||
and parallel.disable_attn_tp_gather
|
||||
)
|
||||
|
||||
|
||||
def _prefill_cp_shards_tokens() -> bool:
|
||||
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
|
||||
parallel = get_parallel()
|
||||
|
||||
@@ -149,7 +149,6 @@ from sglang.srt.utils.common import (
|
||||
add_prefix,
|
||||
get_bool_env_var,
|
||||
rank0_log,
|
||||
require_mlp_sync,
|
||||
set_weight_attrs,
|
||||
)
|
||||
|
||||
@@ -2448,20 +2447,14 @@ def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
|
||||
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the layers build stage boundaries, which is the same for every
|
||||
layer of a stack. The layer's own communication still runs for: a dense
|
||||
MLP sharded over attention TP; an attention-residual bank whose o_proj
|
||||
all-reduce is fused with the pending add, or whose MoE on its attention-TP
|
||||
token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded
|
||||
carry also needs; and SP-MoE on batches that are not padded to a multiple
|
||||
of attention TP (--disable-attn-tp-gather)."""
|
||||
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled():
|
||||
return False
|
||||
layer of a stack. The layer's own communication still runs for: an
|
||||
attention-residual bank whose o_proj all-reduce is fused with the pending
|
||||
add, or whose MoE on its attention-TP token shard (SP-MoE) uses K3's tuned
|
||||
SP collectives, which the sharded carry also needs."""
|
||||
if _fuses_attn_all_reduce(config):
|
||||
return False
|
||||
if not _shards_moe_rows():
|
||||
return True
|
||||
if not require_mlp_sync():
|
||||
return False
|
||||
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ def stub_plan():
|
||||
plan.fusions = None
|
||||
plan._publish_lora_layout = False
|
||||
plan._next_input_rows = None
|
||||
plan._unpadded_attn_tp_size = None
|
||||
plan.paths = {}
|
||||
plan.enters_stack = False
|
||||
return plan
|
||||
|
||||
@@ -19,6 +19,8 @@ from sglang.srt.layers.layer_boundary import (
|
||||
declare_ffn,
|
||||
layer_stack,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import prepare as boundary_prepare
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup, TokenAxis
|
||||
from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_tp_gather_input,
|
||||
keep_output,
|
||||
@@ -397,5 +399,85 @@ class TestPipelineHandoff(CustomTestCase):
|
||||
attention.from_pp(PPProxyTensors({"hidden_states": hidden}), batch)
|
||||
|
||||
|
||||
class TestUnpaddedBatches(CustomTestCase):
|
||||
"""Without attention DP, --disable-attn-tp-gather lets a batch reach the
|
||||
stages with rows that do not divide over attention TP. Such a batch keeps
|
||||
an FFN that would run on this rank's attention-TP slice on the
|
||||
attention's rows instead."""
|
||||
|
||||
def build(self, parallel, *, sparse):
|
||||
with fixture.planning(parallel, a2a=True, boundary_reduction="ar"):
|
||||
with layer_stack():
|
||||
return [s for _ in range(2) for s in layer(sparse=sparse)]
|
||||
|
||||
def test_an_unpadded_batch_keeps_the_attention_rows(self):
|
||||
for sparse in (True, False):
|
||||
with self.subTest(sparse=sparse):
|
||||
parallel = fixture.parallel_of(
|
||||
attn_dp=1,
|
||||
attn_tp=2,
|
||||
disable_attn_tp_gather=True,
|
||||
moe_dense_tp_size=None if sparse else 1,
|
||||
)
|
||||
_, ffn, attention, _ = self.build(parallel, sparse=sparse)
|
||||
ordinary = ffn.plan.paths[BatchVariant.ORDINARY]
|
||||
unpadded = ffn.plan.paths[BatchVariant.UNPADDED]
|
||||
# Rows that divide take the slice; the others complete the
|
||||
# attention's sum on every row and hand on a complete output.
|
||||
self.assertIn(TokenAxis.ATTN_TP, ordinary.entry.input_rows.sharded)
|
||||
self.assertEqual(unpadded.entry.input_rows.sharded, frozenset())
|
||||
self.assertIs(
|
||||
unpadded.entry.prepare.keywords["step"].func,
|
||||
boundary_prepare._reduce_update_read,
|
||||
)
|
||||
self.assertIsNone(unpadded.output.group)
|
||||
self.assertIs(unpadded.output_move, keep_output)
|
||||
# The next attention reads those rows as they are.
|
||||
self.assertIs(
|
||||
attention.plan.paths[BatchVariant.ORDINARY].entry.input_move,
|
||||
attn_tp_gather_input,
|
||||
)
|
||||
self.assertIsNone(
|
||||
attention.plan.paths[BatchVariant.UNPADDED].entry.input_move
|
||||
)
|
||||
|
||||
def test_padded_batches_have_no_unpadded_path(self):
|
||||
for attn_dp, disabled in ((2, True), (1, False)):
|
||||
with self.subTest(attn_dp=attn_dp, disable_attn_tp_gather=disabled):
|
||||
parallel = fixture.parallel_of(
|
||||
attn_dp=attn_dp, attn_tp=2, disable_attn_tp_gather=disabled
|
||||
)
|
||||
_, ffn, _, _ = self.build(parallel, sparse=True)
|
||||
self.assertNotIn(BatchVariant.UNPADDED, ffn.plan.paths)
|
||||
|
||||
|
||||
class TestDenseFfnOverAttentionTp(CustomTestCase):
|
||||
"""A dense FFN sharded over attention TP under attention DP computes on
|
||||
the attention's rows and sums over attention TP, with no DP move."""
|
||||
|
||||
def test_the_ffn_stays_on_the_attention_rows(self):
|
||||
parallel = fixture.parallel_of(attn_dp=2, attn_tp=2)
|
||||
with fixture.planning(parallel, boundary_reduction="ar"), layer_stack():
|
||||
stages = [
|
||||
stage
|
||||
for _ in range(2)
|
||||
for stage in append_stages(
|
||||
(declare_attn(), fixture.Norm()),
|
||||
(declare_ffn(dense_tp_size=2), fixture.Norm()),
|
||||
)
|
||||
]
|
||||
for ffn in stages[1::2]:
|
||||
path = ffn.plan.paths[BatchVariant.ORDINARY]
|
||||
# The attention's sum completes on this rank's rows, ungathered.
|
||||
self.assertIs(
|
||||
path.entry.prepare.keywords["step"].func,
|
||||
boundary_prepare._reduce_update_read,
|
||||
)
|
||||
self.assertEqual(path.output.layout.sharded, {TokenAxis.ATTN_DP})
|
||||
self.assertIs(path.output.group, SumGroup.ATTN_TP)
|
||||
self.assertFalse(path.returns_over_dp)
|
||||
self.assertIs(path.output_move, keep_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -103,6 +103,7 @@ class TestAttentionCpBoundary(CustomTestCase):
|
||||
moe_tp_size=moe_tp_size,
|
||||
dwdp_size=1,
|
||||
enable_attn_tp_input_scattered=False,
|
||||
disable_attn_tp_gather=False,
|
||||
tp_group=group("tp", range(tp_size)),
|
||||
attn_tp_group=group("attn_tp", range(tp_rank, tp_rank + attn_tp_size)),
|
||||
attn_cp_group=group("attn_cp", range(0, tp_size, attn_tp_size)),
|
||||
|
||||
@@ -85,6 +85,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
|
||||
dwdp_size=1,
|
||||
attn_dp_enabled=attn_dp > 1,
|
||||
enable_attn_tp_input_scattered=False,
|
||||
disable_attn_tp_gather=False,
|
||||
tp_group=SimpleNamespace(
|
||||
name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp))
|
||||
),
|
||||
|
||||
@@ -39,6 +39,7 @@ class TestIndependentStageConstruction(CustomTestCase):
|
||||
|
||||
for variant in BatchVariant:
|
||||
plan = stub_plan()
|
||||
plan._unpadded_attn_tp_size = 2
|
||||
ordinary = object()
|
||||
plan.paths[BatchVariant.ORDINARY] = ordinary
|
||||
with (
|
||||
@@ -62,6 +63,11 @@ class TestIndependentStageConstruction(CustomTestCase):
|
||||
"_batch_shards_over_cp",
|
||||
return_value=variant is BatchVariant.CONTEXT_PARALLEL,
|
||||
),
|
||||
patch.object(
|
||||
construction,
|
||||
"_rows_indivisible_over_attn_tp",
|
||||
return_value=variant is BatchVariant.UNPADDED,
|
||||
),
|
||||
):
|
||||
if variant is BatchVariant.ORDINARY:
|
||||
self.assertIs(plan.path_for(None), ordinary)
|
||||
|
||||
@@ -78,7 +78,12 @@ class TestLongcatShortcut(CustomTestCase):
|
||||
),
|
||||
patch_communicator(
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(attn_tp_group=object()),
|
||||
return_value=SimpleNamespace(
|
||||
attn_tp_group=object(),
|
||||
attn_tp_size=tp,
|
||||
attn_dp_enabled=False,
|
||||
disable_attn_tp_gather=False,
|
||||
),
|
||||
),
|
||||
):
|
||||
hidden, _ = layer(
|
||||
|
||||
@@ -66,12 +66,14 @@ def stages(pattern, *, dp=1, tp=1, a2a=False):
|
||||
|
||||
parallel = SimpleNamespace(
|
||||
attn_dp_size=dp,
|
||||
attn_dp_enabled=dp > 1,
|
||||
attn_tp_size=tp,
|
||||
attn_cp_size=1,
|
||||
tp_size=dp * tp,
|
||||
moe_dp_size=1,
|
||||
moe_dense_tp_size=None,
|
||||
enable_attn_tp_input_scattered=False,
|
||||
disable_attn_tp_gather=False,
|
||||
enable_prefill_cp=False,
|
||||
)
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user