[Refactor] Choose the LayerNorm SP region's and input-scattered batches' steps from declarations (#41419)
This commit is contained in:
@@ -100,6 +100,76 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
# The group the layer's input arrives as a partial sum over, if any.
|
||||
input_owes: Optional[SumGroup] = None
|
||||
# Whether the residual is added into one rank's share of the attention
|
||||
# output's sum before that sum completes, instead of after it.
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer while a LayerNorm SP region is active. Its linears
|
||||
all-gather their input and reduce-scatter their output themselves, so the
|
||||
layer's rows, its residual and both outputs stay on each TP rank's slice."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
ffn_output=StageOutput(local),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
hands_on_partial: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on a batch whose attention input is scattered over
|
||||
attention TP. Each layer's input is a partial sum over TP on the full rows:
|
||||
the embedding and every FFN before it leave that sum, and a reduce-scatter
|
||||
completes it onto each rank's slice, which the attention gathers itself.
|
||||
The residual comes back to the full rows inside the attention output's sum."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next layer's input completes the sum; the last layer's FFN does.
|
||||
ffn_output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
input_owes=SumGroup.TP,
|
||||
residual_joins_attention_sum=True,
|
||||
)
|
||||
|
||||
|
||||
def decoder_layer_sides(
|
||||
|
||||
@@ -45,6 +45,8 @@ from sglang.srt.layers.boundary_layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
is_mla_cp_active,
|
||||
@@ -759,6 +761,20 @@ class LayerCommunicator:
|
||||
self._select_declared_boundaries(sides)
|
||||
else:
|
||||
self._select_boundaries_from_scatter_modes()
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
_select_boundary_steps(
|
||||
input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=allow_reduce_scatter
|
||||
and not layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
)
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
@@ -768,15 +784,36 @@ class LayerCommunicator:
|
||||
get_lora().enable_lora
|
||||
)
|
||||
|
||||
# Under LayerNorm SP the residual and norms run on this rank's sequence
|
||||
# shard with no collectives while the region is active.
|
||||
self._sp_region = layernorm_sp.layernorm_sp_enabled()
|
||||
# Under LayerNorm SP, the steps the layer runs while the region is
|
||||
# active; None without SP. The two are exclusive: SP runs a model without
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
|
||||
)
|
||||
if layernorm_sp.layernorm_sp_enabled()
|
||||
else None
|
||||
)
|
||||
|
||||
def _input_can_be_scattered(self) -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on pure TP without an a2a backend. The rest of what
|
||||
``AttnTpContext.init_context`` requires is only known once the model is
|
||||
built."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and parallel.attn_dp_size == 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and get_moe_a2a_backend().is_none()
|
||||
)
|
||||
|
||||
def _declared_sides(self) -> Optional[DecoderLayerSides]:
|
||||
"""The declarations this layer's boundaries are chosen from: an
|
||||
attention and an FFN, without CP, LayerNorm SP or input-scattered
|
||||
attention, in a layer whose previous layer is known. None when the
|
||||
steps come from the scatter modes."""
|
||||
attention and an FFN, without CP, in a layer whose previous layer is
|
||||
known. None when the steps come from the scatter modes. An active
|
||||
LayerNorm SP region and input-scattered attention have their own."""
|
||||
modes = self.layer_scatter_modes
|
||||
parallel = get_parallel()
|
||||
# A MoE dispatched per DP shard computes on this rank's local rows and
|
||||
@@ -785,18 +822,12 @@ class LayerCommunicator:
|
||||
if not (
|
||||
self._takes_declared_boundaries
|
||||
and parallel.attn_cp_size == 1
|
||||
and not layernorm_sp.layernorm_sp_enabled()
|
||||
and not parallel.enable_attn_tp_input_scattered
|
||||
and not enable_moe_dense_fully_dp()
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
|
||||
):
|
||||
return None
|
||||
return decoder_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
},
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
|
||||
previous_on_local_rows=(
|
||||
not modes.is_first_layer
|
||||
@@ -991,13 +1022,18 @@ class LayerCommunicator:
|
||||
hidden_states = owed.partial
|
||||
# The SP region opens at the first layer, re-evaluated per forward so a
|
||||
# crash mid-loop cannot leak into the next one.
|
||||
if self._sp_region and self.layer_scatter_modes.is_first_layer:
|
||||
if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer:
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
if get_attn_tp_context().input_scattered:
|
||||
steps = self._batch_steps()
|
||||
if steps is not None and steps.layer_input is not None:
|
||||
hidden_states, residual = steps.layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
elif get_attn_tp_context().input_scattered:
|
||||
hidden_states, residual = self._tp_reduce_scatter(
|
||||
hidden_states,
|
||||
residual,
|
||||
@@ -1021,17 +1057,29 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_region and get_forward().sp_active
|
||||
return self._sp_steps is not None and get_forward().sp_active
|
||||
|
||||
def _batch_steps(self) -> Optional["BoundarySteps"]:
|
||||
"""The steps this batch runs in place of the layer's ordinary ones: an
|
||||
active LayerNorm SP region's, or input-scattered attention's."""
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps
|
||||
if (
|
||||
self._input_scattered_steps is not None
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
return self._input_scattered_steps
|
||||
return None
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
|
||||
the SP region the attention input stays on this rank's sequence shard."""
|
||||
if not self._in_sp_region():
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
steps = self._batch_steps()
|
||||
move = self._communicate_simple_fn if steps is None else steps.attention_input
|
||||
hidden_states = move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
@@ -1175,7 +1223,7 @@ class LayerCommunicator:
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._in_sp_region():
|
||||
return _mlp_input_norm(
|
||||
return self._sp_steps.ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1185,7 +1233,9 @@ class LayerCommunicator:
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
return self._mlp_input(
|
||||
steps = self._batch_steps()
|
||||
mlp_input = self._mlp_input if steps is None else steps.ffn_input
|
||||
return mlp_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1206,8 +1256,14 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
if self._in_sp_region():
|
||||
return hidden_states, residual
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
@@ -1246,6 +1302,9 @@ class LayerCommunicator:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output.leaves_for_reduce_scatter
|
||||
if not self._ffn_output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if dp_step is not None:
|
||||
@@ -1274,7 +1333,9 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
if not self._ffn_output.leaves_for_next_layer:
|
||||
steps = self._batch_steps()
|
||||
ffn_output = self._ffn_output if steps is None else steps.ffn_output
|
||||
if not ffn_output.leaves_for_next_layer:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
@@ -1843,6 +1904,70 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
return post_experts_reduction_group()
|
||||
|
||||
|
||||
def _token_axis_sizes() -> Dict[TokenAxis, int]:
|
||||
parallel = get_parallel()
|
||||
return {
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
}
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps one set of declarations chooses for a layer's boundaries:
|
||||
into the attention, from the attention output to the FFN input, and the
|
||||
FFN output on to the rows the layer hands on."""
|
||||
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
ffn_output_move: Callable
|
||||
# What the FFN exit reads in place of the layer's own FFN output declaration.
|
||||
ffn_output: StageOutput
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
|
||||
|
||||
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
"""The steps for a batch's own declarations: nothing moves back over
|
||||
attention DP and no fused kernel runs, so the steps do not wait on a
|
||||
per-batch decision."""
|
||||
returns_over_dp, ffn_output_move = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
if sides.input_owes is not None:
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which the
|
||||
# attention takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
sides.input_owes is not SumGroup.TP
|
||||
or rows.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in sides.attention.gathers_itself
|
||||
):
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
ffn_input, _ = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=False,
|
||||
fusions=(),
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
ffn_input=ffn_input,
|
||||
ffn_output_move=ffn_output_move,
|
||||
ffn_output=sides.ffn_output,
|
||||
layer_input=layer_input,
|
||||
)
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
@@ -1866,6 +1991,7 @@ def _select_ffn_input(
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
@@ -1881,7 +2007,7 @@ def _select_ffn_input(
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
@@ -1910,6 +2036,10 @@ def _select_ffn_input(
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm, ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return (
|
||||
partial(
|
||||
@@ -1985,21 +2115,6 @@ def mlp_input_kind(
|
||||
and context.attn_tp_size == 1
|
||||
):
|
||||
return MlpInputKind.NORM
|
||||
if (
|
||||
hidden_in == ScatterMode.SCATTERED
|
||||
and residual_in == ScatterMode.SCATTERED
|
||||
and hidden_out == ScatterMode.SCATTERED
|
||||
and residual_out == ScatterMode.SCATTERED
|
||||
):
|
||||
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
|
||||
# activations stay sequence-sharded across the attn->mlp boundary, so
|
||||
# there is nothing to gather or scatter here -- just the residual add
|
||||
# plus LayerNorm on the local shard. The row-parallel o_proj already
|
||||
# issued the reduce-scatter (g-bar) that the all-reduce would have
|
||||
# done, and the g all-gather is fused into the next column-parallel
|
||||
# linear. Distinct from the branch above because under pure TP
|
||||
# attn_tp_size == tp_size > 1, so that gate does not fire.
|
||||
return MlpInputKind.NORM
|
||||
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
@@ -2080,6 +2195,18 @@ def _mlp_input_scatter(
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_residual_into_sum(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context
|
||||
)
|
||||
|
||||
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -8,7 +8,13 @@ import torch
|
||||
|
||||
import sglang
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
Layout,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
UnreducedOutput,
|
||||
@@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def sp_region_steps():
|
||||
"""The steps a layer runs while a LayerNorm SP region is active."""
|
||||
return comm._select_boundary_steps(
|
||||
sequence_parallel_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: 1,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def make_group(scale=3):
|
||||
"""The group a deferred sum is owed over, with a stubbed all-reduce."""
|
||||
return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale))
|
||||
@@ -51,7 +70,8 @@ def make_communicator(
|
||||
return_value=reduce_scatter
|
||||
)
|
||||
communicator.is_last_layer = False
|
||||
communicator._sp_region = False
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
|
||||
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
|
||||
@@ -284,7 +304,8 @@ class TestSelectFfnCompletion(CustomTestCase):
|
||||
):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._sp_region = sp_region
|
||||
communicator._sp_steps = sp_region_steps() if sp_region else None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters
|
||||
communicator._ffn_output = StageOutput(
|
||||
Layout(frozenset()),
|
||||
|
||||
@@ -123,14 +123,16 @@ class TestBoundarySelection(CustomTestCase):
|
||||
make_context(),
|
||||
Kind.NORM,
|
||||
),
|
||||
((SCATTERED, SCATTERED, SCATTERED, SCATTERED), tp, Kind.NORM),
|
||||
):
|
||||
with self.subTest(layout=layout):
|
||||
self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind)
|
||||
with self.assertRaises(NotImplementedError):
|
||||
comm.mlp_input_kind(
|
||||
modes(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), tp
|
||||
)
|
||||
for layout in (
|
||||
(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL),
|
||||
# The LayerNorm SP region chooses its steps from its declarations.
|
||||
(SCATTERED, SCATTERED, SCATTERED, SCATTERED),
|
||||
):
|
||||
with self.subTest(layout=layout), self.assertRaises(NotImplementedError):
|
||||
comm.mlp_input_kind(modes(*layout), tp)
|
||||
|
||||
def test_postprocess(self):
|
||||
self.assertIs(
|
||||
|
||||
@@ -24,6 +24,8 @@ from sglang.srt.layers.boundary_layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
@@ -410,10 +412,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
def test_layers_that_keep_the_scatter_modes(self):
|
||||
dp = parallel_of(attn_dp=2, attn_tp=2)
|
||||
cases = {
|
||||
"input-scattered attention": (
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True),
|
||||
),
|
||||
"attention CP": (
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=2, attn_tp=1, attn_cp=2),
|
||||
@@ -431,8 +429,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
communicator.allow_deferred_ffn_reduction = True
|
||||
communicator.allow_reduce_scatter = False
|
||||
self.assertIsNone(communicator._declared_sides())
|
||||
# Modes given directly (Nemotron-H's stages, the SP sibling) do not say
|
||||
# which rows the layer takes.
|
||||
# Modes given directly (Nemotron-H's stages) do not say which rows the
|
||||
# layer takes.
|
||||
direct = LayerScatterModes(
|
||||
layer_input_mode=ScatterMode.TP_ATTN_FULL,
|
||||
attn_mode=ScatterMode.TP_ATTN_FULL,
|
||||
@@ -441,10 +439,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
layer_output_mode=ScatterMode.TP_ATTN_FULL,
|
||||
)
|
||||
self.assertFalse(self.declared(build(direct, dp)))
|
||||
with planning(dp, sp=True):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator.layer_scatter_modes = layer_facts(1, 3)
|
||||
self.assertIsNone(communicator._declared_sides())
|
||||
|
||||
def test_subclasses_that_pick_their_own_steps(self):
|
||||
for cls in (
|
||||
@@ -620,6 +614,192 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
|
||||
self.assertIs(communicator._mlp_input_may_return_new_residual, expected)
|
||||
|
||||
|
||||
class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
"""While a LayerNorm SP region is active, the layer runs the steps its
|
||||
region declarations choose: the linears gather and reduce-scatter
|
||||
themselves, so every boundary stays on this rank's slice. Other batches
|
||||
take the layer's ordinary declared steps."""
|
||||
|
||||
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
|
||||
SP_STEPS = comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
|
||||
ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output,
|
||||
)
|
||||
|
||||
def test_the_region_declarations_choose_local_steps(self):
|
||||
for attn_tp in (2, 4):
|
||||
with self.subTest(attn_tp=attn_tp):
|
||||
sides = sequence_parallel_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: 1,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: attn_tp,
|
||||
}
|
||||
)
|
||||
local = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
self.assertEqual(sides.input_rows.sharded, local)
|
||||
self.assertEqual(sides.attention.gathers_itself, local)
|
||||
self.assertEqual(sides.ffn.gathers_itself, local)
|
||||
self.assertIsNone(sides.attention_output.group)
|
||||
self.assertIsNone(sides.ffn_output.group)
|
||||
steps = comm._select_boundary_steps(sides)
|
||||
self.assertEqual(
|
||||
(steps.attention_input, steps.ffn_input, steps.ffn_output_move),
|
||||
(
|
||||
self.SP_STEPS.attention_input,
|
||||
self.SP_STEPS.ffn_input,
|
||||
self.SP_STEPS.ffn_output_move,
|
||||
),
|
||||
)
|
||||
self.assertIsNone(steps.layer_input)
|
||||
|
||||
def test_a_layer_under_sp_takes_both_sets_of_steps(self):
|
||||
parallel = parallel_of(attn_dp=1, attn_tp=2)
|
||||
for layer_id in range(3):
|
||||
with self.subTest(layer_id=layer_id):
|
||||
with planning(parallel, sp=True):
|
||||
communicator = LayerCommunicator(
|
||||
layer_scatter_modes=layer_facts(layer_id, 3),
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
|
||||
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
|
||||
|
||||
def test_without_sp_there_is_no_region(self):
|
||||
with planning(parallel_of(attn_dp=1, attn_tp=2)):
|
||||
communicator = LayerCommunicator(
|
||||
layer_scatter_modes=layer_facts(1, 3),
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertIsNone(communicator._sp_steps)
|
||||
|
||||
def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self):
|
||||
# A sum the FFN exit may hand to the next layer, or a move back over
|
||||
# attention DP, waits on a per-batch decision.
|
||||
for sizes, leaves_next in (
|
||||
(self.SIZES, True),
|
||||
({**self.SIZES, TokenAxis.ATTN_DP: 2}, False),
|
||||
):
|
||||
with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError):
|
||||
comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next))
|
||||
|
||||
|
||||
class TestInputScatteredAttention(CustomTestCase):
|
||||
"""On a batch whose attention input is scattered over attention TP, a layer
|
||||
runs the steps that batch's declarations choose: its input arrives as a TP
|
||||
partial that a reduce-scatter completes onto each rank's slice, and the
|
||||
residual comes back to every row inside the attention output's sum."""
|
||||
|
||||
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
|
||||
|
||||
def test_the_declarations_choose_the_steps(self):
|
||||
for hands_on in (False, True):
|
||||
with self.subTest(hands_on_partial=hands_on):
|
||||
sides = input_scattered_layer_sides(
|
||||
axis_sizes=self.SIZES,
|
||||
ffn_group=SumGroup.TP,
|
||||
hands_on_partial=hands_on,
|
||||
)
|
||||
steps = comm._select_boundary_steps(sides)
|
||||
self.assertIs(steps.layer_input, comm.tp_reduce_scatter)
|
||||
self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial)
|
||||
self.assertIs(steps.ffn_input, comm._mlp_input_residual_into_sum)
|
||||
self.assertIs(
|
||||
steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
)
|
||||
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
|
||||
self.assertFalse(steps.ffn_output.leaves_for_next_layer)
|
||||
|
||||
def test_the_order_of_a_residual_on_the_slice_is_declared(self):
|
||||
# The same layouts take two orders: with a scattered input the residual
|
||||
# joins the attention output's sum; after a MoE on local rows it is
|
||||
# gathered first.
|
||||
sizes = self.SIZES
|
||||
attention = comm.Layout.sharded_over(axis_sizes=sizes)
|
||||
local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
owed = comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True)
|
||||
for joins, step in (
|
||||
(True, comm._mlp_input_residual_into_sum),
|
||||
(False, comm._mlp_input_without_dp),
|
||||
):
|
||||
with self.subTest(residual_joins_sum=joins):
|
||||
steps, _ = comm._select_ffn_input(
|
||||
owed,
|
||||
residual=local,
|
||||
residual_to=attention,
|
||||
need=comm.StageInput(attention),
|
||||
force_layernorm_before_gather=False,
|
||||
fusions=(),
|
||||
residual_joins_sum=joins,
|
||||
)
|
||||
self.assertIs(getattr(steps, "func", steps), step)
|
||||
|
||||
def test_which_layers_can_scatter_their_input(self):
|
||||
configured = dict(enable_attn_tp_input_scattered=True)
|
||||
for name, parallel, a2a, expected in (
|
||||
("pure TP", parallel_of(attn_dp=1, attn_tp=2, **configured), False, True),
|
||||
("not configured", parallel_of(attn_dp=1, attn_tp=2), False, False),
|
||||
(
|
||||
"attention DP",
|
||||
parallel_of(attn_dp=2, attn_tp=2, **configured),
|
||||
False,
|
||||
False,
|
||||
),
|
||||
("a2a", parallel_of(attn_dp=1, attn_tp=2, **configured), True, False),
|
||||
):
|
||||
with self.subTest(name):
|
||||
communicator = build(
|
||||
layer_facts(1, 3, sparse=a2a, previous_sparse=a2a),
|
||||
parallel,
|
||||
a2a=a2a,
|
||||
allow_reduce_scatter=True,
|
||||
)
|
||||
self.assertIs(communicator._input_scattered_steps is not None, expected)
|
||||
|
||||
def test_a_batch_runs_them_only_while_its_input_is_scattered(self):
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
|
||||
)
|
||||
communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
|
||||
for scattered in (False, True):
|
||||
with (
|
||||
self.subTest(input_scattered=scattered),
|
||||
patch.object(
|
||||
comm,
|
||||
"get_attn_tp_context",
|
||||
lambda: SimpleNamespace(input_scattered=scattered),
|
||||
),
|
||||
patch.object(
|
||||
comm, "get_forward", lambda: SimpleNamespace(sp_active=False)
|
||||
),
|
||||
):
|
||||
self.assertIs(
|
||||
communicator._batch_steps(),
|
||||
communicator._input_scattered_steps if scattered else None,
|
||||
)
|
||||
|
||||
def test_the_last_layer_completes_its_own_sum(self):
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
|
||||
)
|
||||
for layer_id, allow, hands_on in (
|
||||
(1, True, True),
|
||||
(2, True, False),
|
||||
(1, False, False),
|
||||
):
|
||||
with self.subTest(layer_id=layer_id, allow_reduce_scatter=allow):
|
||||
communicator = build(
|
||||
layer_facts(layer_id, 3), parallel, allow_reduce_scatter=allow
|
||||
)
|
||||
steps = communicator._input_scattered_steps
|
||||
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Two consecutive layers, rank by rank.
|
||||
|
||||
|
||||
@@ -6,7 +6,13 @@ from unittest.mock import MagicMock, patch
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
Layout,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
@@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def sp_region_steps():
|
||||
"""The steps a layer runs while a LayerNorm SP region is active."""
|
||||
return comm._select_boundary_steps(
|
||||
sequence_parallel_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: 1,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._speculative_algo = None
|
||||
@@ -329,7 +348,8 @@ class TestDeferFfnReduction(CustomTestCase):
|
||||
communicator = _fake_communicator()
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._sp_region = sp_active
|
||||
communicator._sp_steps = sp_region_steps() if sp_active else None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._ffn_output = StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.MOE_OUTPUT,
|
||||
|
||||
@@ -14,6 +14,10 @@ import torch
|
||||
from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
TokenAxis,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
get_flags,
|
||||
@@ -28,6 +32,19 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=9, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def sp_region_steps():
|
||||
"""The steps a layer runs while a LayerNorm SP region is active."""
|
||||
return comm._select_boundary_steps(
|
||||
sequence_parallel_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: 1,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _initialize(*, enable=True, arch="Qwen3ForCausalLM"):
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", enable_layernorm_sp=enable), role="tokenizer"
|
||||
@@ -148,7 +165,8 @@ class TestSpRegionSteps(CustomTestCase):
|
||||
|
||||
def communicator(self, *, first_layer):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c._sp_region = True
|
||||
c._sp_steps = sp_region_steps()
|
||||
c._input_scattered_steps = None
|
||||
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
|
||||
c._attn_input_fusions = ()
|
||||
c.input_layernorm = _Norm()
|
||||
|
||||
@@ -140,7 +140,8 @@ class TestPostprocessReduceScatterv(CustomTestCase):
|
||||
def test_postprocess_passes_the_layer_sparsity(self):
|
||||
seen = {}
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._sp_region = False
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._context = None
|
||||
communicator._postprocess_scatters_to_local_tokens = False
|
||||
communicator.allow_reduce_scatter = False
|
||||
@@ -157,7 +158,8 @@ class TestPostprocessReduceScatterv(CustomTestCase):
|
||||
for step in (object(), None):
|
||||
with self.subTest(reduce_scatter=step is not None):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._sp_region = False
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = True
|
||||
communicator._postprocess_dp_step = lambda forward_batch: step
|
||||
with patch.object(
|
||||
|
||||
@@ -42,7 +42,8 @@ class Norm:
|
||||
def communicator(norm):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c.input_layernorm = norm
|
||||
c._sp_region = False
|
||||
c._sp_steps = None
|
||||
c._input_scattered_steps = None
|
||||
c.qkv_latent_func = None
|
||||
c._context = None
|
||||
c.enable_fused_ar_quant = False
|
||||
|
||||
@@ -222,7 +222,8 @@ def test_communicator_publishes_layout_at_each_transition(
|
||||
communicator._publish_lora_layout = publish_lora_layout
|
||||
communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode)
|
||||
communicator._context = SimpleNamespace()
|
||||
communicator._sp_region = False
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator.post_attention_layernorm = None
|
||||
communicator.input_layernorm = lambda x: x
|
||||
communicator.qkv_latent_func = None
|
||||
|
||||
@@ -57,6 +57,8 @@ class DeferringLayer(nn.Module):
|
||||
self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer
|
||||
self.layer_communicator.is_last_layer = False
|
||||
self.layer_communicator._postprocess_scatters_to_local_tokens = False
|
||||
self.layer_communicator._sp_steps = None
|
||||
self.layer_communicator._input_scattered_steps = None
|
||||
self.layer_communicator.ffn_reduction_group = lambda: GROUP
|
||||
self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = (
|
||||
lambda batch, dp_step: False
|
||||
|
||||
Reference in New Issue
Block a user