[Refactor] Nemotron-H: build each layer's boundaries from its stage and the previous one (#41430)

This commit is contained in:
Cheng Wan
2026-09-27 16:51:27 -07:00
committed by GitHub
parent 6f370c3bcd
commit 8c43c667cb
10 changed files with 560 additions and 136 deletions
+43 -1
View File
@@ -14,7 +14,7 @@
"""Token layouts of the tensors handed across layer communication boundaries."""
from enum import Enum, auto
from typing import FrozenSet, Mapping, Optional
from typing import FrozenSet, Mapping, Optional, Tuple
import msgspec
@@ -108,6 +108,14 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
residual_joins_attention_sum: bool = False
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
@@ -170,6 +178,40 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
)
def stage_edges(
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
) -> Tuple[EdgeDecl, EdgeDecl]:
"""The two boundaries of a layer that is one stage of a sequence of stages:
into it from the previous stage's output (None at the start of the layer
stack) and out of it onto ``rows``, the rows every layer hands on and the
residual is on between stages. The previous output arrives on those rows: a
stage whose output is elsewhere (an FFN on the TP group) moves it back there
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
EdgeDecl(
produced=stage.output,
need=StageInput(rows),
residual=during,
residual_to=rows,
),
)
def sequence_parallel_layer_sides(
*, axis_sizes: Mapping[TokenAxis, int]
) -> DecoderLayerSides:
+143 -16
View File
@@ -725,6 +725,11 @@ class AddAndNorm:
)
def update_and_read_ffn_input(self, hidden_states, residual, norm):
if residual is None:
# The layer stack starts at this FFN: its input is the residual.
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return norm(hidden_states), hidden_states
if hidden_states.shape[0] == 0:
return hidden_states, residual
return norm(hidden_states, residual)
@@ -820,6 +825,9 @@ class LayerCommunicator:
allow_deferred_ffn_reduction: bool = True,
# How the layer writes its residual and reads its stages' inputs.
residual_ops: ResidualOps = ADD_AND_NORM,
# A layer that is one stage of a sequence of stages, instead of an
# attention followed by an FFN.
stage: Optional["LayerStage"] = None,
):
self.layer_scatter_modes = layer_scatter_modes
self.input_layernorm = input_layernorm
@@ -839,6 +847,18 @@ class LayerCommunicator:
)
# The fused kernels every batch's attention input tries first.
self._attn_input_fusions = self._select_attn_input_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
# LoRA kernels need the per-layer token layout only under DP attention.
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
get_lora().enable_lora
)
if stage is not None:
self._init_stage(stage)
return
# Its two boundaries, for a layer that is one stage.
self.stage_edges = None
# The steps the layer's ordinary batches run.
sides = self._declared_sides()
self._declared = sides
@@ -870,20 +890,13 @@ class LayerCommunicator:
if sides is not None and self._input_can_be_scattered()
else None
)
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
# LoRA kernels need the per-layer token layout only under DP attention.
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
get_lora().enable_lora
)
# Under LayerNorm SP, the steps the layer runs while the region is
# active; None without SP. The two are exclusive: SP runs a model without
# q_lora, which input-scattered attention needs.
self._sp_steps = (
_select_boundary_steps(
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()),
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
residual_ops=residual_ops,
attention_fusions=self._attn_input_fusions,
enters_stack=self.layer_scatter_modes.is_first_layer,
@@ -892,12 +905,51 @@ class LayerCommunicator:
else None
)
def _init_stage(self, stage: "LayerStage") -> None:
"""A layer that is one stage: every batch runs the two boundaries its
declarations give."""
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
raise NotImplementedError(
"a layer that is one stage with attention CP or LayerNorm SP"
)
self._declared = None
self._cp_steps = self._input_scattered_steps = self._sp_steps = None
self.stage_edges = stage.edges
into_edge, out_edge = stage.edges
reads_ffn = stage.reads is InputRead.FFN
into = make_boundary(
into_edge,
reads=stage.reads,
fusions=(
self._select_mlp_input_fusions()
if reads_ffn
else self._attn_input_fusions
),
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
residual_ops=self._residual_ops,
enters_stack=stage.enters_stack,
)
out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops)
self._steps = BoundarySteps(
attention_prepare=_another_stage if reads_ffn else into.prepare,
attention_input=_another_stage if reads_ffn else into.input_move,
ffn_input=into.prepare if reads_ffn else _another_stage,
ffn_input_rows=into.input_rows,
ffn_output=out_edge.produced,
ffn_output_move=out.output_move,
ffn_output_move_completes_sum=out.output_move_completes_sum,
ffn_sum_is_movable=out_edge.produced.group is not None,
fused=into.fused,
)
@property
def input_rows(self) -> Layout:
"""The rows the layer's input and residual arrive on in a batch that
runs its ordinary steps."""
if self._declared is not None:
return self._declared.input_rows
if self.stage_edges is not None:
return self.stage_edges[0].residual
# Such a batch holds every token on each CP rank.
return scatter_mode_layouts(
attn_dp_size=self._context.attn_dp_size,
@@ -974,7 +1026,7 @@ class LayerCommunicator:
# Otherwise under CP the FFN completes its own sum.
may_leave = may_leave_to_reduce_scatter = parallel.attn_cp_size == 1
return decoder_layer_sides(
axis_sizes=_token_axis_sizes(cp_active=cp_active),
axis_sizes=token_axis_sizes(cp_active=cp_active),
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
previous_on_local_rows=(
not modes.is_first_layer
@@ -1054,14 +1106,14 @@ class LayerCommunicator:
write-back is not a plain add stays on each rank's slice."""
if self._residual_ops.adds_plainly:
scattered = input_scattered_layer_sides(
axis_sizes=_token_axis_sizes(),
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
)
handoff = _hand_qkv_hook_its_input
else:
scattered = scattered_residual_layer_sides(
axis_sizes=_token_axis_sizes(),
axis_sizes=token_axis_sizes(),
ffn_group=sides.ffn_output.group,
is_first_layer=self.layer_scatter_modes.is_first_layer,
is_last_layer=self.is_last_layer,
@@ -1564,6 +1616,12 @@ class LayerCommunicator:
residual = None
return hidden_states, residual
def mixer_exit(self, forward_batch: ForwardBatch) -> "MixerExit":
"""Decide once whether this stage's mixer (an attention-like stage)
skips its output all-reduce. Use the result as a context manager around
the mixer, then call ``finish``."""
return MixerExit(self, forward_batch)
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
@@ -1722,6 +1780,42 @@ def _leave_to_next_layer(
return wrap(hidden_states), residual
class MixerExit:
"""The scope that publishes a mixer's decision while it runs: inside the
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
row-parallel output projection to skip the all-reduce. It skips when the
stage's output always leaves its sum (to an FFN stage, which completes it in
its input), and when it may leave it and the fused kernel takes it into the
next input norm."""
__slots__ = ("skips_reduction", "_hands_on", "_scope")
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
produced = communicator._batch_steps(forward_batch).ffn_output
self._hands_on = (
produced.leaves_for_next_layer
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
)
self.skips_reduction = produced.always_leaves or self._hands_on
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
def __enter__(self) -> "MixerExit":
self._scope.__enter__()
return self
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor
) -> Union[torch.Tensor, UnreducedOutput]:
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
if self._hands_on:
return UnreducedOutput(hidden_states, group=get_parallel().tp_group)
return hidden_states
class FfnExit:
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
@@ -2135,7 +2229,7 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
return post_experts_reduction_group()
def _token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
"""The token axes' sizes for a batch: attention CP shards tokens only on
a CP extend (``cp_active``); otherwise every CP rank holds them all."""
parallel = get_parallel()
@@ -2151,7 +2245,7 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
rows: from the rows the first overlapped layer takes to the attention's,
and back again for each half."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=_token_axis_sizes()
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
)
pair = CommunicateSummableTensorPairFn
if layer_input_rows == attention:
@@ -2353,6 +2447,11 @@ def _attention_input_step(
)
def _another_stage(*args, **kwargs):
"""The steps of a stage a single-stage layer does not have."""
raise RuntimeError("this layer is one stage and does not have the other")
class InputRead(Enum):
"""How a boundary's consumer reads its input from the residual: with the
attention input norm (prepare_attn) or with the FFN input norm and its
@@ -2362,6 +2461,18 @@ class InputRead(Enum):
FFN = auto()
class LayerStage(msgspec.Struct, frozen=True):
"""A layer that is one stage of a sequence of stages, each an
attention-like mixer or an FFN: how it reads its input, its two boundaries
(into it, and out of it onto the rows every layer hands on, as
``boundary_layout.stage_edges`` gives them), and whether the layer stack
starts at it."""
reads: InputRead
edges: Tuple[EdgeDecl, EdgeDecl]
enters_stack: bool = False
class Boundary(msgspec.Struct, frozen=True):
"""The steps one layer runs at one boundary, chosen from both sides'
declarations. A layer runs the consumer's half of a boundary into one of
@@ -2595,18 +2706,18 @@ def _select_ffn_input(
sliced = need.layout.sharded - produced.layout.sharded
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective.
# completes the attention-TP sum and slices in one collective; a
# complete value is only sliced.
if (
sliced != {TokenAxis.ATTN_TP_SCATTER}
or gathered
or not owes_attention_tp
or residual_to != need.layout
or residual not in (produced.layout, need.layout)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(
_mlp_input_scatter,
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
scatters_residual=residual != residual_to,
residual_ops=residual_ops,
),
@@ -2878,6 +2989,22 @@ def _mlp_input_scatter(
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_slice(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
if scatters_residual and residual is not None:
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
+34 -86
View File
@@ -78,11 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
replace_prefix,
replace_substrings,
)
from sglang.srt.models.nemotron_h_utils import (
feeds_mlp_layer,
is_attn_layer,
make_layer_communicator,
)
from sglang.srt.models.nemotron_h_utils import make_layer_communicator
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.utils import (
@@ -376,66 +372,28 @@ class NemotronHMoE(nn.Module):
class NemotronHMLPLikeDecoderLayer(nn.Module):
"""MLP half of a decoder layer. A preceding Mamba / attention layer hands over
its unreduced output, as between the two halves of a standard decoder layer."""
"""An FFN (MLP / MoE) stage. Its input is the previous stage's output: a
mixer's attention partial sum, or a value that is complete or carries the
sum an FFN before it left."""
def _init_layer_communicator(
self, config: NemotronHConfig, layer_idx: int, *, is_sparse: bool
) -> None:
pattern = config.hybrid_override_pattern
self.input_is_reduced = layer_idx == 0 or not is_attn_layer(
pattern[layer_idx - 1]
)
self.feeds_mlp_layer = feeds_mlp_layer(pattern, layer_idx)
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=False,
allow_reduce_scatter=True,
is_sparse=is_sparse,
is_last_layer=layer_idx == len(pattern) - 1,
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
def forward(
self,
*,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | UnreducedOutput,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.input_is_reduced:
# Fold the reduced input into the residual, leaving prepare_mlp a
# zero update to reduce.
residual = hidden_states if residual is None else hidden_states + residual
hidden_states = torch.zeros_like(hidden_states)
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
# prepare_mlp ignores a deferred reduction, so only defer into a
# Mamba / attention layer.
fuse_mlp_allreduce = (
not self.feeds_mlp_layer
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states = UnreducedOutput(
hidden_states,
group=self.layer_communicator.ffn_reduction_group(forward_batch),
)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return hidden_states, residual
return ffn_exit.finish(hidden_states, residual)
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -469,7 +427,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx, is_sparse=False)
self._init_layer_communicator(config, layer_idx)
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -492,52 +450,37 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx, is_sparse=True)
self._init_layer_communicator(config, layer_idx)
class NemotronHAttnLikeDecoderLayer(nn.Module):
"""Attention half of a decoder layer. Before an MLP layer the mixer leaves its
output unreduced for prepare_mlp; otherwise the mixer reduces it itself."""
"""A mixer (Mamba / attention) stage. Before an FFN stage it leaves its
output all-reduce to that stage's input; otherwise it reduces the output
itself unless the fused kernel takes the sum into the next input norm."""
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
self.feeds_mlp_layer = feeds_mlp_layer(
config.hybrid_override_pattern, layer_idx
)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
def forward(
self,
*,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | UnreducedOutput,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
if forward_batch.forward_mode.is_idle():
return hidden_states, residual
fuse_mlp_allreduce = (
not self.feeds_mlp_layer
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
skip_reduce = self.feeds_mlp_layer or fuse_mlp_allreduce
with get_forward().scoped(fuse_mlp_allreduce=skip_reduce):
with self.layer_communicator.mixer_exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(
hidden_states, forward_batch, skip_reduce
hidden_states, forward_batch, mixer_exit.skips_reduction
)
if fuse_mlp_allreduce:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
return hidden_states, residual
return mixer_exit.finish(hidden_states), residual
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
@@ -779,18 +722,23 @@ class NemotronHModel(nn.Module):
self.layers_to_capture: set[int] = set()
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
pattern = self.config.hybrid_override_pattern
if (
residual is not None
and boundary_idx > 0
and is_attn_layer(pattern[boundary_idx - 1])
and feeds_mlp_layer(pattern, boundary_idx - 1)
):
# Reduce a copy so the MLP layer still receives a TP partial.
if residual is not None and self._owes_attention_sum_at(boundary_idx):
# Reduce a copy so the FFN stage still receives its partial sum.
hidden_states = attn_tp_all_reduce(hidden_states.clone())
# Later norms update the input residual in place.
return hidden_states.clone() if residual is None else hidden_states + residual
def _owes_attention_sum_at(self, boundary_idx: int) -> bool:
"""Whether the value at this boundary is a mixer's partial sum that the
FFN stage after it completes: what the layer after the boundary
declares it takes, or at this rank's end what the last layer declares
it hands on."""
if boundary_idx < self.end_layer:
edge = self.layers[boundary_idx].layer_communicator.stage_edges[0]
else:
edge = self.layers[boundary_idx - 1].layer_communicator.stage_edges[1]
return edge.produced.always_leaves
def forward(
self,
input_ids: torch.Tensor,
@@ -141,7 +141,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
)
self.has_start_projections = has_start_projections
self.has_end_norm = has_end_norm
self.layer_communicator.is_last_layer = True
if has_start_projections:
self.enorm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+95 -15
View File
@@ -1,12 +1,24 @@
"""Layer-communication helpers for the Nemotron-H model."""
from torch import nn
from typing import Optional
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.boundary_layout import (
Layout,
StageDecl,
StageInput,
StageOutput,
SumGroup,
TokenAxis,
stage_edges,
)
from sglang.srt.layers.communicator import (
InputRead,
LayerCommunicator,
LayerScatterModes,
LayerStage,
ScatterMode,
token_axis_sizes,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
@@ -18,9 +30,75 @@ def is_attn_layer(layer_type: str) -> bool:
return layer_type in ATTN_LAYERS
def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool:
next_idx = layer_idx + 1
return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx])
def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]:
"""How the stage at ``layer_idx`` reads its input: a Mamba or attention
mixer like an attention, an MLP or MoE like an FFN; None past either end."""
if not 0 <= layer_idx < len(pattern):
return None
return InputRead.ATTENTION if is_attn_layer(pattern[layer_idx]) else InputRead.FFN
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
"""What the stage at ``layer_idx`` declares. A mixer computes on the
attention's rows and leaves its attention-TP sum to an FFN stage after it,
or, when a fused kernel takes it, to a mixer after it. An FFN computes on
the TP group's rows, a MoE dispatched by an a2a backend on this rank's own,
and may leave its sum to a mixer after it."""
axis_sizes = token_axis_sizes()
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
following = _stage_kind(pattern, layer_idx + 1)
if is_attn_layer(pattern[layer_idx]):
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return StageDecl(
StageInput(attention),
StageOutput(
attention,
group=SumGroup.ATTN_TP if owes else None,
always_leaves=owes and following is InputRead.FFN,
leaves_for_next_layer=owes and following is InputRead.ATTENTION,
),
)
sparse = pattern[layer_idx] == MOE
if sparse and not get_moe_a2a_backend().is_none():
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return StageDecl(StageInput(local), StageOutput(local))
full = Layout.sharded_over(axis_sizes=axis_sizes)
return StageDecl(
StageInput(full),
StageOutput(
full,
group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
leaves_for_next_layer=following is InputRead.ATTENTION,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
),
)
def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
"""The layer's stage and its two boundaries, from its own declaration and
the previous stage's."""
rows = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
)
return LayerStage(
reads=_stage_kind(pattern, layer_idx),
edges=stage_edges(
previous=(
_stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None
),
stage=_stage_decl(pattern, layer_idx),
rows=rows,
),
enters_stack=layer_idx == 0,
)
def _build_layer_scatter_modes(
@@ -43,19 +121,21 @@ def _build_layer_scatter_modes(
def make_layer_communicator(
layer_norm: RMSNorm,
*,
for_attn: bool,
allow_reduce_scatter: bool = False,
is_sparse: bool = False,
is_last_layer: bool = False,
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
) -> LayerCommunicator:
"""The communicator of a layer that is one stage: only its own norm, and
boundaries built from the stages next to it in the pattern."""
stage = layer_stage(pattern, layer_idx)
for_attn = stage.reads is InputRead.ATTENTION
return LayerCommunicator(
layer_scatter_modes=_build_layer_scatter_modes(is_sparse, is_last_layer),
input_layernorm=layer_norm if for_attn else nn.Identity(),
post_attention_layernorm=nn.Identity() if for_attn else layer_norm,
layer_scatter_modes=_build_layer_scatter_modes(
pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1
),
input_layernorm=layer_norm if for_attn else None,
post_attention_layernorm=None if for_attn else layer_norm,
# With attention TP > 1, the default gather adds the residual to one
# rank's partial in bf16 before the cross-rank sum.
force_layernorm_before_dp_gather=True,
allow_reduce_scatter=allow_reduce_scatter,
allow_reduce_scatter=not for_attn,
stage=stage,
)