[Refactor] Choose the LayerNorm SP region's and input-scattered batches' steps from declarations (#41419)
This commit is contained in:
@@ -100,6 +100,76 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
# The group the layer's input arrives as a partial sum over, if any.
|
||||
input_owes: Optional[SumGroup] = None
|
||||
# Whether the residual is added into one rank's share of the attention
|
||||
# output's sum before that sum completes, instead of after it.
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer while a LayerNorm SP region is active. Its linears
|
||||
all-gather their input and reduce-scatter their output themselves, so the
|
||||
layer's rows, its residual and both outputs stay on each TP rank's slice."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
ffn_output=StageOutput(local),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
hands_on_partial: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on a batch whose attention input is scattered over
|
||||
attention TP. Each layer's input is a partial sum over TP on the full rows:
|
||||
the embedding and every FFN before it leave that sum, and a reduce-scatter
|
||||
completes it onto each rank's slice, which the attention gathers itself.
|
||||
The residual comes back to the full rows inside the attention output's sum."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next layer's input completes the sum; the last layer's FFN does.
|
||||
ffn_output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
input_owes=SumGroup.TP,
|
||||
residual_joins_attention_sum=True,
|
||||
)
|
||||
|
||||
|
||||
def decoder_layer_sides(
|
||||
|
||||
@@ -45,6 +45,8 @@ from sglang.srt.layers.boundary_layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
is_mla_cp_active,
|
||||
@@ -759,6 +761,20 @@ class LayerCommunicator:
|
||||
self._select_declared_boundaries(sides)
|
||||
else:
|
||||
self._select_boundaries_from_scatter_modes()
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
_select_boundary_steps(
|
||||
input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=allow_reduce_scatter
|
||||
and not layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
)
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
@@ -768,15 +784,36 @@ class LayerCommunicator:
|
||||
get_lora().enable_lora
|
||||
)
|
||||
|
||||
# Under LayerNorm SP the residual and norms run on this rank's sequence
|
||||
# shard with no collectives while the region is active.
|
||||
self._sp_region = layernorm_sp.layernorm_sp_enabled()
|
||||
# Under LayerNorm SP, the steps the layer runs while the region is
|
||||
# active; None without SP. The two are exclusive: SP runs a model without
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
|
||||
)
|
||||
if layernorm_sp.layernorm_sp_enabled()
|
||||
else None
|
||||
)
|
||||
|
||||
def _input_can_be_scattered(self) -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on pure TP without an a2a backend. The rest of what
|
||||
``AttnTpContext.init_context`` requires is only known once the model is
|
||||
built."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and parallel.attn_dp_size == 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and get_moe_a2a_backend().is_none()
|
||||
)
|
||||
|
||||
def _declared_sides(self) -> Optional[DecoderLayerSides]:
|
||||
"""The declarations this layer's boundaries are chosen from: an
|
||||
attention and an FFN, without CP, LayerNorm SP or input-scattered
|
||||
attention, in a layer whose previous layer is known. None when the
|
||||
steps come from the scatter modes."""
|
||||
attention and an FFN, without CP, in a layer whose previous layer is
|
||||
known. None when the steps come from the scatter modes. An active
|
||||
LayerNorm SP region and input-scattered attention have their own."""
|
||||
modes = self.layer_scatter_modes
|
||||
parallel = get_parallel()
|
||||
# A MoE dispatched per DP shard computes on this rank's local rows and
|
||||
@@ -785,18 +822,12 @@ class LayerCommunicator:
|
||||
if not (
|
||||
self._takes_declared_boundaries
|
||||
and parallel.attn_cp_size == 1
|
||||
and not layernorm_sp.layernorm_sp_enabled()
|
||||
and not parallel.enable_attn_tp_input_scattered
|
||||
and not enable_moe_dense_fully_dp()
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
|
||||
):
|
||||
return None
|
||||
return decoder_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
},
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
|
||||
previous_on_local_rows=(
|
||||
not modes.is_first_layer
|
||||
@@ -991,13 +1022,18 @@ class LayerCommunicator:
|
||||
hidden_states = owed.partial
|
||||
# The SP region opens at the first layer, re-evaluated per forward so a
|
||||
# crash mid-loop cannot leak into the next one.
|
||||
if self._sp_region and self.layer_scatter_modes.is_first_layer:
|
||||
if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer:
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
if get_attn_tp_context().input_scattered:
|
||||
steps = self._batch_steps()
|
||||
if steps is not None and steps.layer_input is not None:
|
||||
hidden_states, residual = steps.layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
elif get_attn_tp_context().input_scattered:
|
||||
hidden_states, residual = self._tp_reduce_scatter(
|
||||
hidden_states,
|
||||
residual,
|
||||
@@ -1021,17 +1057,29 @@ class LayerCommunicator:
|
||||
)
|
||||
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_region and get_forward().sp_active
|
||||
return self._sp_steps is not None and get_forward().sp_active
|
||||
|
||||
def _batch_steps(self) -> Optional["BoundarySteps"]:
|
||||
"""The steps this batch runs in place of the layer's ordinary ones: an
|
||||
active LayerNorm SP region's, or input-scattered attention's."""
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps
|
||||
if (
|
||||
self._input_scattered_steps is not None
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
return self._input_scattered_steps
|
||||
return None
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
|
||||
the SP region the attention input stays on this rank's sequence shard."""
|
||||
if not self._in_sp_region():
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
steps = self._batch_steps()
|
||||
move = self._communicate_simple_fn if steps is None else steps.attention_input
|
||||
hidden_states = move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
@@ -1175,7 +1223,7 @@ class LayerCommunicator:
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._in_sp_region():
|
||||
return _mlp_input_norm(
|
||||
return self._sp_steps.ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1185,7 +1233,9 @@ class LayerCommunicator:
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
return self._mlp_input(
|
||||
steps = self._batch_steps()
|
||||
mlp_input = self._mlp_input if steps is None else steps.ffn_input
|
||||
return mlp_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1206,8 +1256,14 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
if self._in_sp_region():
|
||||
return hidden_states, residual
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
@@ -1246,6 +1302,9 @@ class LayerCommunicator:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output.leaves_for_reduce_scatter
|
||||
if not self._ffn_output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if dp_step is not None:
|
||||
@@ -1274,7 +1333,9 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
if not self._ffn_output.leaves_for_next_layer:
|
||||
steps = self._batch_steps()
|
||||
ffn_output = self._ffn_output if steps is None else steps.ffn_output
|
||||
if not ffn_output.leaves_for_next_layer:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
@@ -1843,6 +1904,70 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
return post_experts_reduction_group()
|
||||
|
||||
|
||||
def _token_axis_sizes() -> Dict[TokenAxis, int]:
|
||||
parallel = get_parallel()
|
||||
return {
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
}
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps one set of declarations chooses for a layer's boundaries:
|
||||
into the attention, from the attention output to the FFN input, and the
|
||||
FFN output on to the rows the layer hands on."""
|
||||
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
ffn_output_move: Callable
|
||||
# What the FFN exit reads in place of the layer's own FFN output declaration.
|
||||
ffn_output: StageOutput
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
|
||||
|
||||
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
"""The steps for a batch's own declarations: nothing moves back over
|
||||
attention DP and no fused kernel runs, so the steps do not wait on a
|
||||
per-batch decision."""
|
||||
returns_over_dp, ffn_output_move = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
if sides.input_owes is not None:
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which the
|
||||
# attention takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
sides.input_owes is not SumGroup.TP
|
||||
or rows.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in sides.attention.gathers_itself
|
||||
):
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
ffn_input, _ = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=False,
|
||||
fusions=(),
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
ffn_input=ffn_input,
|
||||
ffn_output_move=ffn_output_move,
|
||||
ffn_output=sides.ffn_output,
|
||||
layer_input=layer_input,
|
||||
)
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
@@ -1866,6 +1991,7 @@ def _select_ffn_input(
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
@@ -1881,7 +2007,7 @@ def _select_ffn_input(
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
@@ -1910,6 +2036,10 @@ def _select_ffn_input(
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm, ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return (
|
||||
partial(
|
||||
@@ -1985,21 +2115,6 @@ def mlp_input_kind(
|
||||
and context.attn_tp_size == 1
|
||||
):
|
||||
return MlpInputKind.NORM
|
||||
if (
|
||||
hidden_in == ScatterMode.SCATTERED
|
||||
and residual_in == ScatterMode.SCATTERED
|
||||
and hidden_out == ScatterMode.SCATTERED
|
||||
and residual_out == ScatterMode.SCATTERED
|
||||
):
|
||||
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
|
||||
# activations stay sequence-sharded across the attn->mlp boundary, so
|
||||
# there is nothing to gather or scatter here -- just the residual add
|
||||
# plus LayerNorm on the local shard. The row-parallel o_proj already
|
||||
# issued the reduce-scatter (g-bar) that the all-reduce would have
|
||||
# done, and the g all-gather is fused into the next column-parallel
|
||||
# linear. Distinct from the branch above because under pure TP
|
||||
# attn_tp_size == tp_size > 1, so that gate does not fire.
|
||||
return MlpInputKind.NORM
|
||||
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
|
||||
ScatterMode.SCATTERED,
|
||||
ScatterMode.TP_ATTN_FULL,
|
||||
@@ -2080,6 +2195,18 @@ def _mlp_input_scatter(
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_residual_into_sum(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context
|
||||
)
|
||||
|
||||
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user