[Refactor] Nemotron-H: build each layer's boundaries from its stage and the previous one (#41430)
This commit is contained in:
@@ -14,7 +14,7 @@
|
||||
"""Token layouts of the tensors handed across layer communication boundaries."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from typing import FrozenSet, Mapping, Optional
|
||||
from typing import FrozenSet, Mapping, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -108,6 +108,14 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
@@ -170,6 +178,40 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
)
|
||||
|
||||
|
||||
def stage_edges(
|
||||
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
|
||||
) -> Tuple[EdgeDecl, EdgeDecl]:
|
||||
"""The two boundaries of a layer that is one stage of a sequence of stages:
|
||||
into it from the previous stage's output (None at the start of the layer
|
||||
stack) and out of it onto ``rows``, the rows every layer hands on and the
|
||||
residual is on between stages. The previous output arrives on those rows: a
|
||||
stage whose output is elsewhere (an FFN on the TP group) moves it back there
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
|
||||
EdgeDecl(
|
||||
produced=stage.output,
|
||||
need=StageInput(rows),
|
||||
residual=during,
|
||||
residual_to=rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
|
||||
@@ -725,6 +725,11 @@ class AddAndNorm:
|
||||
)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
if residual is None:
|
||||
# The layer stack starts at this FFN: its input is the residual.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return norm(hidden_states), hidden_states
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
@@ -820,6 +825,9 @@ class LayerCommunicator:
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
# How the layer writes its residual and reads its stages' inputs.
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
# A layer that is one stage of a sequence of stages, instead of an
|
||||
# attention followed by an FFN.
|
||||
stage: Optional["LayerStage"] = None,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -839,6 +847,18 @@ class LayerCommunicator:
|
||||
)
|
||||
# The fused kernels every batch's attention input tries first.
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
# LoRA kernels need the per-layer token layout only under DP attention.
|
||||
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
||||
get_lora().enable_lora
|
||||
)
|
||||
if stage is not None:
|
||||
self._init_stage(stage)
|
||||
return
|
||||
# Its two boundaries, for a layer that is one stage.
|
||||
self.stage_edges = None
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
self._declared = sides
|
||||
@@ -870,20 +890,13 @@ class LayerCommunicator:
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
# LoRA kernels need the per-layer token layout only under DP attention.
|
||||
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
||||
get_lora().enable_lora
|
||||
)
|
||||
|
||||
# Under LayerNorm SP, the steps the layer runs while the region is
|
||||
# active; None without SP. The two are exclusive: SP runs a model without
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()),
|
||||
sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()),
|
||||
residual_ops=residual_ops,
|
||||
attention_fusions=self._attn_input_fusions,
|
||||
enters_stack=self.layer_scatter_modes.is_first_layer,
|
||||
@@ -892,12 +905,51 @@ class LayerCommunicator:
|
||||
else None
|
||||
)
|
||||
|
||||
def _init_stage(self, stage: "LayerStage") -> None:
|
||||
"""A layer that is one stage: every batch runs the two boundaries its
|
||||
declarations give."""
|
||||
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
|
||||
raise NotImplementedError(
|
||||
"a layer that is one stage with attention CP or LayerNorm SP"
|
||||
)
|
||||
self._declared = None
|
||||
self._cp_steps = self._input_scattered_steps = self._sp_steps = None
|
||||
self.stage_edges = stage.edges
|
||||
into_edge, out_edge = stage.edges
|
||||
reads_ffn = stage.reads is InputRead.FFN
|
||||
into = make_boundary(
|
||||
into_edge,
|
||||
reads=stage.reads,
|
||||
fusions=(
|
||||
self._select_mlp_input_fusions()
|
||||
if reads_ffn
|
||||
else self._attn_input_fusions
|
||||
),
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
residual_ops=self._residual_ops,
|
||||
enters_stack=stage.enters_stack,
|
||||
)
|
||||
out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops)
|
||||
self._steps = BoundarySteps(
|
||||
attention_prepare=_another_stage if reads_ffn else into.prepare,
|
||||
attention_input=_another_stage if reads_ffn else into.input_move,
|
||||
ffn_input=into.prepare if reads_ffn else _another_stage,
|
||||
ffn_input_rows=into.input_rows,
|
||||
ffn_output=out_edge.produced,
|
||||
ffn_output_move=out.output_move,
|
||||
ffn_output_move_completes_sum=out.output_move_completes_sum,
|
||||
ffn_sum_is_movable=out_edge.produced.group is not None,
|
||||
fused=into.fused,
|
||||
)
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the layer's input and residual arrive on in a batch that
|
||||
runs its ordinary steps."""
|
||||
if self._declared is not None:
|
||||
return self._declared.input_rows
|
||||
if self.stage_edges is not None:
|
||||
return self.stage_edges[0].residual
|
||||
# Such a batch holds every token on each CP rank.
|
||||
return scatter_mode_layouts(
|
||||
attn_dp_size=self._context.attn_dp_size,
|
||||
@@ -974,7 +1026,7 @@ class LayerCommunicator:
|
||||
# Otherwise under CP the FFN completes its own sum.
|
||||
may_leave = may_leave_to_reduce_scatter = parallel.attn_cp_size == 1
|
||||
return decoder_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(cp_active=cp_active),
|
||||
axis_sizes=token_axis_sizes(cp_active=cp_active),
|
||||
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
|
||||
previous_on_local_rows=(
|
||||
not modes.is_first_layer
|
||||
@@ -1054,14 +1106,14 @@ class LayerCommunicator:
|
||||
write-back is not a plain add stays on each rank's slice."""
|
||||
if self._residual_ops.adds_plainly:
|
||||
scattered = input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
|
||||
)
|
||||
handoff = _hand_qkv_hook_its_input
|
||||
else:
|
||||
scattered = scattered_residual_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
axis_sizes=token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
is_first_layer=self.layer_scatter_modes.is_first_layer,
|
||||
is_last_layer=self.is_last_layer,
|
||||
@@ -1564,6 +1616,12 @@ class LayerCommunicator:
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def mixer_exit(self, forward_batch: ForwardBatch) -> "MixerExit":
|
||||
"""Decide once whether this stage's mixer (an attention-like stage)
|
||||
skips its output all-reduce. Use the result as a context manager around
|
||||
the mixer, then call ``finish``."""
|
||||
return MixerExit(self, forward_batch)
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
@@ -1722,6 +1780,42 @@ def _leave_to_next_layer(
|
||||
return wrap(hidden_states), residual
|
||||
|
||||
|
||||
class MixerExit:
|
||||
"""The scope that publishes a mixer's decision while it runs: inside the
|
||||
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
|
||||
row-parallel output projection to skip the all-reduce. It skips when the
|
||||
stage's output always leaves its sum (to an FFN stage, which completes it in
|
||||
its input), and when it may leave it and the fused kernel takes it into the
|
||||
next input norm."""
|
||||
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope")
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
produced = communicator._batch_steps(forward_batch).ffn_output
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
)
|
||||
self.skips_reduction = produced.always_leaves or self._hands_on
|
||||
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
|
||||
|
||||
def __enter__(self) -> "MixerExit":
|
||||
self._scope.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> Union[torch.Tensor, UnreducedOutput]:
|
||||
"""The mixer's output: its partial sum to the FFN stage after it, as a
|
||||
value that owes the sum to a mixer after it, or complete."""
|
||||
if self._hands_on:
|
||||
return UnreducedOutput(hidden_states, group=get_parallel().tp_group)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FfnExit:
|
||||
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
|
||||
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
|
||||
@@ -2135,7 +2229,7 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
return post_experts_reduction_group()
|
||||
|
||||
|
||||
def _token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
|
||||
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
|
||||
"""The token axes' sizes for a batch: attention CP shards tokens only on
|
||||
a CP extend (``cp_active``); otherwise every CP rank holds them all."""
|
||||
parallel = get_parallel()
|
||||
@@ -2151,7 +2245,7 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
rows: from the rows the first overlapped layer takes to the attention's,
|
||||
and back again for each half."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=_token_axis_sizes()
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
if layer_input_rows == attention:
|
||||
@@ -2353,6 +2447,11 @@ def _attention_input_step(
|
||||
)
|
||||
|
||||
|
||||
def _another_stage(*args, **kwargs):
|
||||
"""The steps of a stage a single-stage layer does not have."""
|
||||
raise RuntimeError("this layer is one stage and does not have the other")
|
||||
|
||||
|
||||
class InputRead(Enum):
|
||||
"""How a boundary's consumer reads its input from the residual: with the
|
||||
attention input norm (prepare_attn) or with the FFN input norm and its
|
||||
@@ -2362,6 +2461,18 @@ class InputRead(Enum):
|
||||
FFN = auto()
|
||||
|
||||
|
||||
class LayerStage(msgspec.Struct, frozen=True):
|
||||
"""A layer that is one stage of a sequence of stages, each an
|
||||
attention-like mixer or an FFN: how it reads its input, its two boundaries
|
||||
(into it, and out of it onto the rows every layer hands on, as
|
||||
``boundary_layout.stage_edges`` gives them), and whether the layer stack
|
||||
starts at it."""
|
||||
|
||||
reads: InputRead
|
||||
edges: Tuple[EdgeDecl, EdgeDecl]
|
||||
enters_stack: bool = False
|
||||
|
||||
|
||||
class Boundary(msgspec.Struct, frozen=True):
|
||||
"""The steps one layer runs at one boundary, chosen from both sides'
|
||||
declarations. A layer runs the consumer's half of a boundary into one of
|
||||
@@ -2595,18 +2706,18 @@ def _select_ffn_input(
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective.
|
||||
# completes the attention-TP sum and slices in one collective; a
|
||||
# complete value is only sliced.
|
||||
if (
|
||||
sliced != {TokenAxis.ATTN_TP_SCATTER}
|
||||
or gathered
|
||||
or not owes_attention_tp
|
||||
or residual_to != need.layout
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
|
||||
scatters_residual=residual != residual_to,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
@@ -2878,6 +2989,22 @@ def _mlp_input_scatter(
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_slice(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -78,11 +78,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
replace_prefix,
|
||||
replace_substrings,
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import (
|
||||
feeds_mlp_layer,
|
||||
is_attn_layer,
|
||||
make_layer_communicator,
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import make_layer_communicator
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
@@ -376,66 +372,28 @@ class NemotronHMoE(nn.Module):
|
||||
|
||||
|
||||
class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
"""MLP half of a decoder layer. A preceding Mamba / attention layer hands over
|
||||
its unreduced output, as between the two halves of a standard decoder layer."""
|
||||
"""An FFN (MLP / MoE) stage. Its input is the previous stage's output: a
|
||||
mixer's attention partial sum, or a value that is complete or carries the
|
||||
sum an FFN before it left."""
|
||||
|
||||
def _init_layer_communicator(
|
||||
self, config: NemotronHConfig, layer_idx: int, *, is_sparse: bool
|
||||
) -> None:
|
||||
pattern = config.hybrid_override_pattern
|
||||
self.input_is_reduced = layer_idx == 0 or not is_attn_layer(
|
||||
pattern[layer_idx - 1]
|
||||
)
|
||||
self.feeds_mlp_layer = feeds_mlp_layer(pattern, layer_idx)
|
||||
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=False,
|
||||
allow_reduce_scatter=True,
|
||||
is_sparse=is_sparse,
|
||||
is_last_layer=layer_idx == len(pattern) - 1,
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | UnreducedOutput,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.input_is_reduced:
|
||||
# Fold the reduced input into the residual, leaving prepare_mlp a
|
||||
# zero update to reduce.
|
||||
residual = hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states = torch.zeros_like(hidden_states)
|
||||
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
# prepare_mlp ignores a deferred reduction, so only defer into a
|
||||
# Mamba / attention layer.
|
||||
fuse_mlp_allreduce = (
|
||||
not self.feeds_mlp_layer
|
||||
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states,
|
||||
group=self.layer_communicator.ffn_reduction_group(forward_batch),
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
return hidden_states, residual
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
|
||||
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -469,7 +427,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx, is_sparse=False)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
|
||||
|
||||
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -492,52 +450,37 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx, is_sparse=True)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
|
||||
|
||||
class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
"""Attention half of a decoder layer. Before an MLP layer the mixer leaves its
|
||||
output unreduced for prepare_mlp; otherwise the mixer reduces it itself."""
|
||||
"""A mixer (Mamba / attention) stage. Before an FFN stage it leaves its
|
||||
output all-reduce to that stage's input; otherwise it reduces the output
|
||||
itself unless the fused kernel takes the sum into the next input norm."""
|
||||
|
||||
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.feeds_mlp_layer = feeds_mlp_layer(
|
||||
config.hybrid_override_pattern, layer_idx
|
||||
)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor | UnreducedOutput,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, residual
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
not self.feeds_mlp_layer
|
||||
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
skip_reduce = self.feeds_mlp_layer or fuse_mlp_allreduce
|
||||
with get_forward().scoped(fuse_mlp_allreduce=skip_reduce):
|
||||
with self.layer_communicator.mixer_exit(forward_batch) as mixer_exit:
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, skip_reduce
|
||||
hidden_states, forward_batch, mixer_exit.skips_reduction
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
return hidden_states, residual
|
||||
return mixer_exit.finish(hidden_states), residual
|
||||
|
||||
|
||||
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
@@ -779,18 +722,23 @@ class NemotronHModel(nn.Module):
|
||||
self.layers_to_capture: set[int] = set()
|
||||
|
||||
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
|
||||
pattern = self.config.hybrid_override_pattern
|
||||
if (
|
||||
residual is not None
|
||||
and boundary_idx > 0
|
||||
and is_attn_layer(pattern[boundary_idx - 1])
|
||||
and feeds_mlp_layer(pattern, boundary_idx - 1)
|
||||
):
|
||||
# Reduce a copy so the MLP layer still receives a TP partial.
|
||||
if residual is not None and self._owes_attention_sum_at(boundary_idx):
|
||||
# Reduce a copy so the FFN stage still receives its partial sum.
|
||||
hidden_states = attn_tp_all_reduce(hidden_states.clone())
|
||||
# Later norms update the input residual in place.
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
|
||||
def _owes_attention_sum_at(self, boundary_idx: int) -> bool:
|
||||
"""Whether the value at this boundary is a mixer's partial sum that the
|
||||
FFN stage after it completes: what the layer after the boundary
|
||||
declares it takes, or at this rank's end what the last layer declares
|
||||
it hands on."""
|
||||
if boundary_idx < self.end_layer:
|
||||
edge = self.layers[boundary_idx].layer_communicator.stage_edges[0]
|
||||
else:
|
||||
edge = self.layers[boundary_idx - 1].layer_communicator.stage_edges[1]
|
||||
return edge.produced.always_leaves
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -141,7 +141,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
)
|
||||
self.has_start_projections = has_start_projections
|
||||
self.has_end_norm = has_end_norm
|
||||
self.layer_communicator.is_last_layer = True
|
||||
|
||||
if has_start_projections:
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
"""Layer-communication helpers for the Nemotron-H model."""
|
||||
|
||||
from torch import nn
|
||||
from typing import Optional
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
Layout,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
stage_edges,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
InputRead,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerStage,
|
||||
ScatterMode,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
@@ -18,9 +30,75 @@ def is_attn_layer(layer_type: str) -> bool:
|
||||
return layer_type in ATTN_LAYERS
|
||||
|
||||
|
||||
def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool:
|
||||
next_idx = layer_idx + 1
|
||||
return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx])
|
||||
def _stage_kind(pattern: str, layer_idx: int) -> Optional[InputRead]:
|
||||
"""How the stage at ``layer_idx`` reads its input: a Mamba or attention
|
||||
mixer like an attention, an MLP or MoE like an FFN; None past either end."""
|
||||
if not 0 <= layer_idx < len(pattern):
|
||||
return None
|
||||
return InputRead.ATTENTION if is_attn_layer(pattern[layer_idx]) else InputRead.FFN
|
||||
|
||||
|
||||
def _stage_decl(pattern: str, layer_idx: int) -> StageDecl:
|
||||
"""What the stage at ``layer_idx`` declares. A mixer computes on the
|
||||
attention's rows and leaves its attention-TP sum to an FFN stage after it,
|
||||
or, when a fused kernel takes it, to a mixer after it. An FFN computes on
|
||||
the TP group's rows, a MoE dispatched by an a2a backend on this rank's own,
|
||||
and may leave its sum to a mixer after it."""
|
||||
axis_sizes = token_axis_sizes()
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
following = _stage_kind(pattern, layer_idx + 1)
|
||||
if is_attn_layer(pattern[layer_idx]):
|
||||
owes = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return StageDecl(
|
||||
StageInput(attention),
|
||||
StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
always_leaves=owes and following is InputRead.FFN,
|
||||
leaves_for_next_layer=owes and following is InputRead.ATTENTION,
|
||||
),
|
||||
)
|
||||
sparse = pattern[layer_idx] == MOE
|
||||
if sparse and not get_moe_a2a_backend().is_none():
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return StageDecl(StageInput(local), StageOutput(local))
|
||||
full = Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
return StageDecl(
|
||||
StageInput(full),
|
||||
StageOutput(
|
||||
full,
|
||||
group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=following is InputRead.ATTENTION,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
|
||||
"""The layer's stage and its two boundaries, from its own declaration and
|
||||
the previous stage's."""
|
||||
rows = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
return LayerStage(
|
||||
reads=_stage_kind(pattern, layer_idx),
|
||||
edges=stage_edges(
|
||||
previous=(
|
||||
_stage_decl(pattern, layer_idx - 1).output if layer_idx > 0 else None
|
||||
),
|
||||
stage=_stage_decl(pattern, layer_idx),
|
||||
rows=rows,
|
||||
),
|
||||
enters_stack=layer_idx == 0,
|
||||
)
|
||||
|
||||
|
||||
def _build_layer_scatter_modes(
|
||||
@@ -43,19 +121,21 @@ def _build_layer_scatter_modes(
|
||||
|
||||
|
||||
def make_layer_communicator(
|
||||
layer_norm: RMSNorm,
|
||||
*,
|
||||
for_attn: bool,
|
||||
allow_reduce_scatter: bool = False,
|
||||
is_sparse: bool = False,
|
||||
is_last_layer: bool = False,
|
||||
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
|
||||
) -> LayerCommunicator:
|
||||
"""The communicator of a layer that is one stage: only its own norm, and
|
||||
boundaries built from the stages next to it in the pattern."""
|
||||
stage = layer_stage(pattern, layer_idx)
|
||||
for_attn = stage.reads is InputRead.ATTENTION
|
||||
return LayerCommunicator(
|
||||
layer_scatter_modes=_build_layer_scatter_modes(is_sparse, is_last_layer),
|
||||
input_layernorm=layer_norm if for_attn else nn.Identity(),
|
||||
post_attention_layernorm=nn.Identity() if for_attn else layer_norm,
|
||||
layer_scatter_modes=_build_layer_scatter_modes(
|
||||
pattern[layer_idx] == MOE, is_last_layer=layer_idx == len(pattern) - 1
|
||||
),
|
||||
input_layernorm=layer_norm if for_attn else None,
|
||||
post_attention_layernorm=None if for_attn else layer_norm,
|
||||
# With attention TP > 1, the default gather adds the residual to one
|
||||
# rank's partial in bf16 before the cross-rank sum.
|
||||
force_layernorm_before_dp_gather=True,
|
||||
allow_reduce_scatter=allow_reduce_scatter,
|
||||
allow_reduce_scatter=not for_attn,
|
||||
stage=stage,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user