[Refactor] Split the layer communicator into a package (move only) (#41439)
This commit is contained in:
@@ -14,9 +14,9 @@ from sglang.srt.batch_overlap.operations import (
|
||||
)
|
||||
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.boundary_layout import Layout
|
||||
from sglang.srt.layers.communicator import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
reduce_output,
|
||||
tbo_split_moves,
|
||||
)
|
||||
|
||||
@@ -1,403 +0,0 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Token layouts of the tensors handed across layer communication boundaries."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from typing import FrozenSet, Mapping, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class TokenAxis(Enum):
|
||||
"""A parallel dimension across which ranks hold different tokens."""
|
||||
|
||||
ATTN_DP = auto()
|
||||
ATTN_CP = auto()
|
||||
# Each attention-TP rank holds a slice of its group's tokens.
|
||||
ATTN_TP_SCATTER = auto()
|
||||
|
||||
|
||||
class Layout(msgspec.Struct, frozen=True):
|
||||
"""The token axes a rank's rows are sharded over. Single-rank axes are left
|
||||
out, so two layouts are equal exactly when every rank holds the same tokens."""
|
||||
|
||||
sharded: FrozenSet[TokenAxis]
|
||||
|
||||
@classmethod
|
||||
def sharded_over(
|
||||
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> "Layout":
|
||||
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
|
||||
|
||||
|
||||
class SumGroup(Enum):
|
||||
"""The group a stage's output is summed over, by name: the handle comes
|
||||
from get_parallel() when the sum runs."""
|
||||
|
||||
ATTN_TP = auto()
|
||||
TP = auto()
|
||||
# The group one all-reduce of a MoE output runs over.
|
||||
MOE_OUTPUT = auto()
|
||||
|
||||
|
||||
class StageInput(msgspec.Struct, frozen=True):
|
||||
"""The rows a stage's consumer needs: sharded over the token axes its
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
"""What a stage's producer hands the boundary after it, fixed at
|
||||
construction: its rows, the group its output is summed over, and when it
|
||||
leaves that sum to the boundary instead of completing it.
|
||||
|
||||
What one output actually owes is carried with the value: a plain
|
||||
tensor handed across layers is complete, an ``UnreducedOutput`` names what
|
||||
is left. A raw compute output entering a boundary is read with this
|
||||
declaration and the boundary's decision for the batch."""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
group: Optional[SumGroup] = None
|
||||
# The producer never reduces its output, e.g. a row-parallel projection
|
||||
# built with reduce_results=False.
|
||||
always_leaves: bool = False
|
||||
# Otherwise it leaves the sum only when the boundary publishes the flag for
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
leaves_for_next_layer: bool = False
|
||||
leaves_for_reduce_scatter: bool = False
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
|
||||
|
||||
class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""What a decoder layer declares about its attention, its FFN and the rows
|
||||
between them; ``decoder_layer_edges`` turns it into the layer's
|
||||
boundaries."""
|
||||
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageInput
|
||||
attention_output: StageOutput
|
||||
ffn: StageInput
|
||||
ffn_output: StageOutput
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
# The group the layer's input arrives as a partial sum over, if any.
|
||||
input_owes: Optional[SumGroup] = None
|
||||
# Whether the residual is added into one rank's share of the attention
|
||||
# output's sum before that sum completes, instead of after it.
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
rows the residual is on before and after the boundary."""
|
||||
|
||||
produced: StageOutput
|
||||
need: StageInput
|
||||
residual: Layout
|
||||
residual_to: Layout
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
|
||||
|
||||
class DecoderLayerEdges(msgspec.Struct, frozen=True):
|
||||
"""A decoder layer's three boundaries: from the rows the previous layer
|
||||
handed on into the attention, from the attention into the FFN, and from
|
||||
the FFN onto the rows the layer hands on. The next layer runs the rest of
|
||||
the last one."""
|
||||
|
||||
into_attention: EdgeDecl
|
||||
into_ffn: EdgeDecl
|
||||
out_of_ffn: EdgeDecl
|
||||
|
||||
|
||||
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
"""The boundaries of a decoder layer, each with the declarations of its two
|
||||
sides. What arrives at the layer is complete unless it owes a sum by
|
||||
construction (``input_owes``); a sum the previous layer leaves for a batch
|
||||
comes with the value."""
|
||||
owes = sides.input_owes is not None
|
||||
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
attention_rows = (
|
||||
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
|
||||
if owes
|
||||
else sides.input_rows
|
||||
)
|
||||
return DecoderLayerEdges(
|
||||
into_attention=EdgeDecl(
|
||||
produced=arrived,
|
||||
need=sides.attention,
|
||||
residual=sides.input_rows,
|
||||
residual_to=attention_rows,
|
||||
),
|
||||
into_ffn=EdgeDecl(
|
||||
produced=sides.attention_output,
|
||||
need=sides.ffn,
|
||||
residual=attention_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
),
|
||||
out_of_ffn=EdgeDecl(
|
||||
produced=sides.ffn_output,
|
||||
need=StageInput(sides.output_rows),
|
||||
residual=sides.ffn_residual_rows,
|
||||
residual_to=sides.output_rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def stage_edges(
|
||||
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
|
||||
) -> Tuple[EdgeDecl, EdgeDecl]:
|
||||
"""The two boundaries of a layer that is one stage of a sequence of stages:
|
||||
into it from the previous stage's output (None at the start of the layer
|
||||
stack) and out of it onto ``rows``, the rows every layer hands on and the
|
||||
residual is on between stages. The previous output arrives on those rows: a
|
||||
stage whose output is elsewhere (an FFN on the TP group) moves it back there
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
|
||||
EdgeDecl(
|
||||
produced=stage.output,
|
||||
need=StageInput(rows),
|
||||
residual=during,
|
||||
residual_to=rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer while a LayerNorm SP region is active. Its linears
|
||||
all-gather their input and reduce-scatter their output themselves, so the
|
||||
layer's rows, its residual and both outputs stay on each TP rank's slice."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
ffn_output=StageOutput(local),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
|
||||
|
||||
def scattered_residual_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
is_first_layer: bool,
|
||||
is_last_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on an input-scattered batch whose residual stays on
|
||||
each attention-TP rank's slice (MHC). The attention and the FFN compute on
|
||||
the full rows. The attention takes the slice and gathers it itself (its
|
||||
QKV hook, or the handoff before an attention that needs the rows); the
|
||||
boundary brings the FFN its rows and moves both outputs back onto the
|
||||
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
|
||||
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
|
||||
is the embedding's partial sum on the full rows; the last layer hands on
|
||||
the full rows again."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
ffn_output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
input_owes=SumGroup.TP if is_first_layer else None,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
hands_on_partial: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on a batch whose attention input is scattered over
|
||||
attention TP. Each layer's input is a partial sum over TP on the full rows:
|
||||
the embedding and every FFN before it leave that sum, and a reduce-scatter
|
||||
completes it onto each rank's slice, which the attention gathers itself.
|
||||
The residual comes back to the full rows inside the attention output's sum."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next layer's input completes the sum; the last layer's FFN does.
|
||||
ffn_output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
input_owes=SumGroup.TP,
|
||||
residual_joins_attention_sum=True,
|
||||
)
|
||||
|
||||
|
||||
def decoder_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_on_local_rows: bool,
|
||||
previous_on_local_rows: bool,
|
||||
is_last_layer: bool,
|
||||
attention_gathers_local_rows: bool,
|
||||
ffn_group: SumGroup,
|
||||
leaves_for_next_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
hands_on_attention_rows: bool = False,
|
||||
ffn_shards_over_cp: bool = False,
|
||||
) -> DecoderLayerSides:
|
||||
"""An attention followed by an FFN, derived from the groups each computes
|
||||
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
|
||||
DP shard, which completes its own combine, or a dense MLP on every rank).
|
||||
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
|
||||
computes each CP shard on its own ranks: its rows stay sharded over CP.
|
||||
A layer on local rows hands on the attention's rows when it is the last, or
|
||||
with ``hands_on_attention_rows``."""
|
||||
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
# Each attention-TP rank's own slice of those rows.
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
# The TP group spans every token axis, so an FFN on it needs every row.
|
||||
ffn = (
|
||||
local
|
||||
if ffn_on_local_rows
|
||||
else Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
)
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if attention_tp else None,
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
ffn=StageInput(ffn),
|
||||
ffn_output=(
|
||||
StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
group=ffn_group,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
)
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=(
|
||||
local
|
||||
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
|
||||
else attention
|
||||
),
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,124 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Communication between a decoder layer's stages."""
|
||||
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
DecoderLayerSides,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
InputRead,
|
||||
LayerStage,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
decoder_layer_edges,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
make_boundary,
|
||||
sequence_parallel_layer_sides,
|
||||
stage_edges,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layer import (
|
||||
FfnCompletion,
|
||||
FfnExit,
|
||||
FfnExitFusion,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
MHCLayerCommunicator,
|
||||
MixerExit,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
ScatterMode,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
enable_moe_dense_fully_dp,
|
||||
moe_cp_gathers_sparse_moe_input,
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
layer_input_buffer,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD_AND_NORM,
|
||||
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import (
|
||||
MHCState,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ADD_AND_NORM",
|
||||
"AttentionInputs",
|
||||
"BoundarySteps",
|
||||
"CommunicateContext",
|
||||
"CommunicateSimpleFn",
|
||||
"CommunicateSummableTensorPairFn",
|
||||
"DecoderLayerSides",
|
||||
"EdgeDecl",
|
||||
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
|
||||
"FfnCompletion",
|
||||
"FfnExit",
|
||||
"FfnExitFusion",
|
||||
"FusedMlpInput",
|
||||
"HandoffOutput",
|
||||
"InputRead",
|
||||
"LayerCommunicator",
|
||||
"LayerScatterModes",
|
||||
"LayerStage",
|
||||
"Layout",
|
||||
"MHCLayerCommunicator",
|
||||
"MHCState",
|
||||
"MixerExit",
|
||||
"ScatterMode",
|
||||
"StageDecl",
|
||||
"StageInput",
|
||||
"StageOutput",
|
||||
"SumGroup",
|
||||
"TokenAxis",
|
||||
"UnreducedOutput",
|
||||
"decoder_layer_edges",
|
||||
"decoder_layer_sides",
|
||||
"enable_moe_dense_fully_dp",
|
||||
"get_attn_tp_context",
|
||||
"input_scattered_layer_sides",
|
||||
"layer_input_buffer",
|
||||
"make_boundary",
|
||||
"moe_cp_gathers_sparse_moe_input",
|
||||
"move_rows",
|
||||
"reduce_output",
|
||||
"sequence_parallel_layer_sides",
|
||||
"sparse_moe_gathers_over_moe_cp",
|
||||
"stage_edges",
|
||||
"tbo_split_moves",
|
||||
"token_axis_sizes",
|
||||
"tp_reduce_scatter",
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""What the communicator takes from the modules it serves."""
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""The attention's inputs: the attention-TP context, its cached inputs, and
|
||||
the attention-TP gather and slice."""
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
get_local_dp_buffer,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel, get_spec
|
||||
from sglang.srt.utils import is_cuda, is_npu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
class AttentionInputs:
|
||||
def __init__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Callable,
|
||||
*,
|
||||
is_pre_gathered: bool = False,
|
||||
):
|
||||
self.hidden_states_local = hidden_states
|
||||
self.forward_batch = forward_batch
|
||||
self.qkv_latent_func = qkv_latent_func
|
||||
self.hidden_states_ = None
|
||||
self.qkv_latent_ = None
|
||||
# When True, hidden_states_local is already attn_tp-gathered upstream
|
||||
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
|
||||
self.is_pre_gathered = is_pre_gathered
|
||||
|
||||
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
def fetch_qkv_latent(self):
|
||||
if self.qkv_latent_ is not None:
|
||||
return self.qkv_latent_
|
||||
assert self.qkv_latent_func is not None
|
||||
self.qkv_latent_ = self.qkv_latent_func(
|
||||
self.hidden_states_local, self.forward_batch
|
||||
)
|
||||
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
||||
self.qkv_latent_ = self.tp_all_gather_hidden_states(
|
||||
self.qkv_latent_, self.forward_batch
|
||||
)
|
||||
return self.qkv_latent_
|
||||
|
||||
def fetch_hidden_states(self):
|
||||
if self.hidden_states_ is not None:
|
||||
return self.hidden_states_
|
||||
self.hidden_states_ = self.hidden_states_local
|
||||
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
||||
self.hidden_states_ = self.tp_all_gather_hidden_states(
|
||||
self.hidden_states_, self.forward_batch
|
||||
)
|
||||
return self.hidden_states_
|
||||
|
||||
|
||||
class AttnTpContext:
|
||||
def __init__(self):
|
||||
self.allow_input_scattered = False
|
||||
self.is_dsa = False
|
||||
|
||||
def init_context(self, q_lora_rank, is_dsa, is_mhc=False):
|
||||
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
|
||||
# cannot use scattered inputs.
|
||||
self.is_dsa = is_dsa
|
||||
self.allow_input_scattered = (
|
||||
get_parallel().enable_attn_tp_input_scattered
|
||||
and (_is_cuda or _is_npu)
|
||||
and q_lora_rank is not None
|
||||
and (is_mhc or not is_dsa)
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not enable_moe_dense_fully_dp()
|
||||
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
and get_spec().speculative_algorithm != "EAGLE3"
|
||||
)
|
||||
if get_parallel().enable_attn_tp_input_scattered:
|
||||
if not self.allow_input_scattered:
|
||||
logging.info(
|
||||
"attn_tp_input_scattered is not enabled while other conditions are not met"
|
||||
)
|
||||
else:
|
||||
logging.info("attn_tp_input_scattered is enabled")
|
||||
|
||||
def use_input_scattered(self, forward_batch: ForwardBatch):
|
||||
return (
|
||||
self.allow_input_scattered
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.input_ids is not None
|
||||
and not forward_batch.can_run_tbo
|
||||
)
|
||||
|
||||
@property
|
||||
def input_scattered(self):
|
||||
return get_forward().attn_input_scattered
|
||||
|
||||
def set_attn_inputs(self, attn_inputs: AttentionInputs):
|
||||
get_forward().set("attn_inputs", attn_inputs)
|
||||
|
||||
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
if attn_inputs is not None:
|
||||
attn_inputs.hidden_states_local = hidden_states
|
||||
|
||||
def fetch_qkv_latent(self):
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
assert attn_inputs is not None
|
||||
return attn_inputs.fetch_qkv_latent()
|
||||
|
||||
def fetch_hidden_states(self):
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
assert attn_inputs is not None
|
||||
return attn_inputs.fetch_hidden_states()
|
||||
|
||||
def clear_attn_inputs(self) -> None:
|
||||
get_forward().set("attn_inputs", None)
|
||||
|
||||
@contextmanager
|
||||
def maybe_input_scattered(self, forward_batch: ForwardBatch):
|
||||
flag = self.use_input_scattered(forward_batch)
|
||||
forward = get_forward()
|
||||
# scoped() also restores when the forward raises — the old in-place
|
||||
# swap leaked the flag on exceptions.
|
||||
with forward.scoped(attn_input_scattered=flag):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
forward.set("attn_inputs", None)
|
||||
|
||||
|
||||
ATTN_TP_CONTEXT = AttnTpContext()
|
||||
|
||||
|
||||
def get_attn_tp_context():
|
||||
return ATTN_TP_CONTEXT
|
||||
|
||||
|
||||
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
||||
gathered = get_local_dp_buffer(
|
||||
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(gathered, tensor)
|
||||
return gathered
|
||||
|
||||
|
||||
def _redistribute_to_attn_tp_shards(
|
||||
tensor: torch.Tensor, context: CommunicateContext
|
||||
) -> torch.Tensor:
|
||||
return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
@@ -0,0 +1,969 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""What each stage produces and needs, and the boundary steps chosen from those
|
||||
declarations."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from functools import partial
|
||||
from typing import Callable, FrozenSet, Mapping, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_gathers_over_attention_cp,
|
||||
_same_ranks,
|
||||
_sum_group,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
_hand_qkv_hook_its_input,
|
||||
_mlp_input_completing_owed,
|
||||
_mlp_input_dp_partial,
|
||||
_mlp_input_dp_replicate,
|
||||
_mlp_input_gather_attention_cp,
|
||||
_mlp_input_gather_moe_cp,
|
||||
_mlp_input_norm,
|
||||
_mlp_input_on_residual_shard,
|
||||
_mlp_input_residual_into_sum,
|
||||
_mlp_input_scatter,
|
||||
_mlp_input_slice,
|
||||
_mlp_input_without_dp,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import ResidualOps
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class StageInput(msgspec.Struct, frozen=True):
|
||||
"""The rows a stage's consumer needs: sharded over the token axes its
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
"""What a stage's producer hands the boundary after it, fixed at
|
||||
construction: its rows, the group its output is summed over, and when it
|
||||
leaves that sum to the boundary instead of completing it.
|
||||
|
||||
What one output actually owes is carried with the value: a plain
|
||||
tensor handed across layers is complete, an ``UnreducedOutput`` names what
|
||||
is left. A raw compute output entering a boundary is read with this
|
||||
declaration and the boundary's decision for the batch."""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
group: Optional[SumGroup] = None
|
||||
# The producer never reduces its output, e.g. a row-parallel projection
|
||||
# built with reduce_results=False.
|
||||
always_leaves: bool = False
|
||||
# Otherwise it leaves the sum only when the boundary publishes the flag for
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
leaves_for_next_layer: bool = False
|
||||
leaves_for_reduce_scatter: bool = False
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
|
||||
|
||||
class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""What a decoder layer declares about its attention, its FFN and the rows
|
||||
between them; ``decoder_layer_edges`` turns it into the layer's
|
||||
boundaries."""
|
||||
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageInput
|
||||
attention_output: StageOutput
|
||||
ffn: StageInput
|
||||
ffn_output: StageOutput
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
# The group the layer's input arrives as a partial sum over, if any.
|
||||
input_owes: Optional[SumGroup] = None
|
||||
# Whether the residual is added into one rank's share of the attention
|
||||
# output's sum before that sum completes, instead of after it.
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
rows the residual is on before and after the boundary."""
|
||||
|
||||
produced: StageOutput
|
||||
need: StageInput
|
||||
residual: Layout
|
||||
residual_to: Layout
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
|
||||
|
||||
class DecoderLayerEdges(msgspec.Struct, frozen=True):
|
||||
"""A decoder layer's three boundaries: from the rows the previous layer
|
||||
handed on into the attention, from the attention into the FFN, and from
|
||||
the FFN onto the rows the layer hands on. The next layer runs the rest of
|
||||
the last one."""
|
||||
|
||||
into_attention: EdgeDecl
|
||||
into_ffn: EdgeDecl
|
||||
out_of_ffn: EdgeDecl
|
||||
|
||||
|
||||
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
"""The boundaries of a decoder layer, each with the declarations of its two
|
||||
sides. What arrives at the layer is complete unless it owes a sum by
|
||||
construction (``input_owes``); a sum the previous layer leaves for a batch
|
||||
comes with the value."""
|
||||
owes = sides.input_owes is not None
|
||||
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
attention_rows = (
|
||||
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
|
||||
if owes
|
||||
else sides.input_rows
|
||||
)
|
||||
return DecoderLayerEdges(
|
||||
into_attention=EdgeDecl(
|
||||
produced=arrived,
|
||||
need=sides.attention,
|
||||
residual=sides.input_rows,
|
||||
residual_to=attention_rows,
|
||||
),
|
||||
into_ffn=EdgeDecl(
|
||||
produced=sides.attention_output,
|
||||
need=sides.ffn,
|
||||
residual=attention_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
),
|
||||
out_of_ffn=EdgeDecl(
|
||||
produced=sides.ffn_output,
|
||||
need=StageInput(sides.output_rows),
|
||||
residual=sides.ffn_residual_rows,
|
||||
residual_to=sides.output_rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def stage_edges(
|
||||
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
|
||||
) -> Tuple[EdgeDecl, EdgeDecl]:
|
||||
"""The two boundaries of a layer that is one stage of a sequence of stages:
|
||||
into it from the previous stage's output (None at the start of the layer
|
||||
stack) and out of it onto ``rows``, the rows every layer hands on and the
|
||||
residual is on between stages. The previous output arrives on those rows: a
|
||||
stage whose output is elsewhere (an FFN on the TP group) moves it back there
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
|
||||
EdgeDecl(
|
||||
produced=stage.output,
|
||||
need=StageInput(rows),
|
||||
residual=during,
|
||||
residual_to=rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer while a LayerNorm SP region is active. Its linears
|
||||
all-gather their input and reduce-scatter their output themselves, so the
|
||||
layer's rows, its residual and both outputs stay on each TP rank's slice."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather the slices into their GEMM.
|
||||
attention=StageInput(attention, gathers_itself=gathers),
|
||||
# o_proj and down reduce-scatter out of theirs, whether fused or not.
|
||||
attention_output=StageOutput(local),
|
||||
ffn=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
ffn_output=StageOutput(local),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
|
||||
|
||||
def scattered_residual_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
is_first_layer: bool,
|
||||
is_last_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on an input-scattered batch whose residual stays on
|
||||
each attention-TP rank's slice (MHC). The attention and the FFN compute on
|
||||
the full rows. The attention takes the slice and gathers it itself (its
|
||||
QKV hook, or the handoff before an attention that needs the rows); the
|
||||
boundary brings the FFN its rows and moves both outputs back onto the
|
||||
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
|
||||
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
|
||||
is the embedding's partial sum on the full rows; the last layer hands on
|
||||
the full rows again."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
ffn_output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
input_owes=SumGroup.TP if is_first_layer else None,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
hands_on_partial: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on a batch whose attention input is scattered over
|
||||
attention TP. Each layer's input is a partial sum over TP on the full rows:
|
||||
the embedding and every FFN before it leave that sum, and a reduce-scatter
|
||||
completes it onto each rank's slice, which the attention gathers itself.
|
||||
The residual comes back to the full rows inside the attention output's sum."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next layer's input completes the sum; the last layer's FFN does.
|
||||
ffn_output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
input_owes=SumGroup.TP,
|
||||
residual_joins_attention_sum=True,
|
||||
)
|
||||
|
||||
|
||||
def decoder_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_on_local_rows: bool,
|
||||
previous_on_local_rows: bool,
|
||||
is_last_layer: bool,
|
||||
attention_gathers_local_rows: bool,
|
||||
ffn_group: SumGroup,
|
||||
leaves_for_next_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
hands_on_attention_rows: bool = False,
|
||||
ffn_shards_over_cp: bool = False,
|
||||
) -> DecoderLayerSides:
|
||||
"""An attention followed by an FFN, derived from the groups each computes
|
||||
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
|
||||
DP shard, which completes its own combine, or a dense MLP on every rank).
|
||||
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
|
||||
computes each CP shard on its own ranks: its rows stay sharded over CP.
|
||||
A layer on local rows hands on the attention's rows when it is the last, or
|
||||
with ``hands_on_attention_rows``."""
|
||||
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
# Each attention-TP rank's own slice of those rows.
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
# The TP group spans every token axis, so an FFN on it needs every row.
|
||||
ffn = (
|
||||
local
|
||||
if ffn_on_local_rows
|
||||
else Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
)
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if attention_tp else None,
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
ffn=StageInput(ffn),
|
||||
ffn_output=(
|
||||
StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
group=ffn_group,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
)
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=(
|
||||
local
|
||||
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
|
||||
else attention
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
"""A kernel that completes the sum the attention output owes together with
|
||||
the residual add and the post-attention norm, in that order.
|
||||
|
||||
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
|
||||
the residual, or None when it does not take the batch; it returns None only
|
||||
before touching its inputs or starting a collective."""
|
||||
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
# It may hand back a new residual and leave the one it took unchanged.
|
||||
may_return_new_residual: bool
|
||||
|
||||
|
||||
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
"""The moves around the two-batch-overlap split, which cuts the attention's
|
||||
rows: from the rows the first overlapped layer takes to the attention's,
|
||||
and back again for each half."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
if layer_input_rows == attention:
|
||||
return pair._trivial, pair._trivial
|
||||
if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice: write the residual in and gather over attention
|
||||
# TP, then take the slice of each half.
|
||||
return pair._gather, pair._scatter
|
||||
raise NotImplementedError(f"{layer_input_rows=}")
|
||||
|
||||
|
||||
class CpMoves(msgspec.Struct, frozen=True):
|
||||
"""How a CP extend's rows reach an FFN that needs all of them and come back,
|
||||
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
|
||||
after each rank has completed its own block, ``take_back`` returns this
|
||||
rank's block of a complete output, and ``reduce_scatter``, where there is
|
||||
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
|
||||
returns the block in the same collective."""
|
||||
|
||||
gather: Callable
|
||||
take_back: Callable
|
||||
reduce_scatter: Optional[Callable] = None
|
||||
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
|
||||
|
||||
|
||||
def _cp_moves() -> CpMoves:
|
||||
"""DSA and MLA CP gather equal shards over the attention-CP group and can
|
||||
complete a sum over it. GQA prefill CP gathers blocks padded to the longest
|
||||
over the MoE-CP group and takes back only a complete output."""
|
||||
if _gathers_over_attention_cp():
|
||||
return CpMoves(
|
||||
gather=_mlp_input_gather_attention_cp,
|
||||
take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard,
|
||||
reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp,
|
||||
reduce_scatter_group=lambda: get_parallel().attn_cp_group,
|
||||
)
|
||||
return CpMoves(
|
||||
gather=_mlp_input_gather_moe_cp,
|
||||
take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
||||
)
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
rows the layer hands on."""
|
||||
|
||||
# The half into the attention: completes what the input owes, writes the
|
||||
# previous output into the residual and reads the attention input
|
||||
# (_attention_input_step); attention_input then moves it.
|
||||
attention_prepare: Callable
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
# The rows ffn_input hands the FFN.
|
||||
ffn_input_rows: Layout
|
||||
# What the FFN exit reads: the FFN output's group and what it may leave.
|
||||
ffn_output: StageOutput
|
||||
# The postprocess that moves the FFN output on; None when it goes back over
|
||||
# attention DP, whose step the FFN exit and postprocess choose per batch.
|
||||
ffn_output_move: Optional[Callable]
|
||||
# Whether the next layer's input can take the FFN's sum.
|
||||
ffn_sum_is_movable: bool
|
||||
# Whether ffn_output_move also completes the sum the FFN leaves.
|
||||
ffn_output_move_completes_sum: bool = False
|
||||
# The fused kernels ffn_input tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Hands the attention its input once attention_input has moved it:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
return self.ffn_output_move is None
|
||||
|
||||
|
||||
def _attention_input_step(
|
||||
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
context: "CommunicateContext",
|
||||
*,
|
||||
quant_format: str,
|
||||
post_residual_addition: Optional[torch.Tensor],
|
||||
layer_input: Optional[Callable],
|
||||
fusions: Tuple[Callable, ...],
|
||||
enters_stack: bool,
|
||||
residual_ops: ResidualOps,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A boundary's half into an attention: complete what the previous layer
|
||||
left (a value that owes a sum, or a producer's handoff one of ``fusions``
|
||||
consumes with the add and norm), what the input owes by construction
|
||||
(``layer_input``), then write the previous output into the residual and read
|
||||
the attention input with ``norm``. The layer stack's first layer
|
||||
(``enters_stack``) starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
|
||||
if owed is not None and residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if isinstance(owed, UnreducedOutput) and owed.reduce_and_redistribute is not None:
|
||||
# No fused kernel runs under attention DP: the reduce-scatter back to
|
||||
# this rank's tokens comes first.
|
||||
hidden_states, owed = reduce_output(owed), None
|
||||
if owed is not None:
|
||||
for fused in fusions:
|
||||
result = fused(owed, residual, forward_batch, post_residual_addition)
|
||||
if result is not None:
|
||||
return result
|
||||
if isinstance(owed, HandoffOutput):
|
||||
# No fused kernel took the handoff: its producer completes it.
|
||||
hidden_states, owed = reduce_output(owed), None
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
if enters:
|
||||
hidden_states, residual = residual_ops.enter(hidden_states), None
|
||||
if owed is not None and hidden_states.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return residual_ops.read_attention_input(hidden_states, norm, quant_format)
|
||||
return residual_ops.update_and_read_attention_input(
|
||||
hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
)
|
||||
|
||||
|
||||
def _another_stage(*args, **kwargs):
|
||||
"""The steps of a stage a single-stage layer does not have."""
|
||||
raise RuntimeError("this layer is one stage and does not have the other")
|
||||
|
||||
|
||||
class InputRead(Enum):
|
||||
"""How a boundary's consumer reads its input from the residual: with the
|
||||
attention input norm (prepare_attn) or with the FFN input norm and its
|
||||
fused kernels (prepare_mlp)."""
|
||||
|
||||
ATTENTION = auto()
|
||||
FFN = auto()
|
||||
|
||||
|
||||
class LayerStage(msgspec.Struct, frozen=True):
|
||||
"""A layer that is one stage of a sequence of stages, each an
|
||||
attention-like mixer or an FFN: how it reads its input, its two boundaries
|
||||
(into it, and out of it onto the rows every layer hands on, as
|
||||
``stage_edges`` gives them), and whether the layer stack
|
||||
starts at it."""
|
||||
|
||||
reads: InputRead
|
||||
edges: Tuple[EdgeDecl, EdgeDecl]
|
||||
enters_stack: bool = False
|
||||
|
||||
|
||||
class Boundary(msgspec.Struct, frozen=True):
|
||||
"""The steps one layer runs at one boundary, chosen from both sides'
|
||||
declarations. A layer runs the consumer's half of a boundary into one of
|
||||
its stages, and the producer's half of the boundary after its last stage;
|
||||
the neighbouring layer runs the other half of that one."""
|
||||
|
||||
edge: EdgeDecl
|
||||
# The consumer's half: completing what the input owes, the add and the
|
||||
# norm; into the FFN also the move onto the rows it needs.
|
||||
prepare: Optional[Callable] = None
|
||||
# Into an attention, the move onto its rows after prepare.
|
||||
input_move: Optional[Callable] = None
|
||||
# The fused kernels an FFN's prepare tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# The producer's half: the postprocess that moves the output onto the rows
|
||||
# the layer hands on; None when it goes back over attention DP, whose step
|
||||
# the FFN exit and postprocess choose per batch.
|
||||
output_move: Optional[Callable] = None
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the consumer is handed: what it needs, still sharded over
|
||||
the axes it gathers itself."""
|
||||
need = self.edge.need
|
||||
return Layout(
|
||||
need.layout.sharded
|
||||
| (self.edge.produced.layout.sharded & need.gathers_itself)
|
||||
)
|
||||
|
||||
|
||||
def make_boundary(
|
||||
edge: EdgeDecl,
|
||||
*,
|
||||
reads: Optional[InputRead],
|
||||
fusions: Tuple = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""The steps a layer runs at ``edge``, around the residual operations;
|
||||
``cp_moves`` for an edge that gathers over or returns across attention CP.
|
||||
``reads`` is how the consumer reads its input, or None when the consumer
|
||||
runs in the next layer and ``edge.need`` is the rows this layer hands on:
|
||||
then only the producer's half runs here. ``fusions`` are the fused kernels
|
||||
the consumer tries first (FusedMlpInput into an FFN, the attention input's
|
||||
entries into an attention); ``enters_stack`` for the edge into the layer
|
||||
stack's first attention. The consumer's half reads only this edge's
|
||||
declarations, never what the producer chose for a batch; a sum left for a
|
||||
batch arrives with the value."""
|
||||
if reads is None:
|
||||
if edge.need.layout != edge.residual_to:
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
output_move=None if returns_over_dp else output_move,
|
||||
output_move_completes_sum=completes_sum,
|
||||
)
|
||||
if reads is InputRead.FFN:
|
||||
input_step, fused = _select_ffn_input(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return Boundary(edge, prepare=input_step, fused=fused)
|
||||
layer_input = None
|
||||
if edge.produced.always_leaves:
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which the
|
||||
# attention takes: its group is the TP group without attention DP or CP.
|
||||
if (
|
||||
edge.produced.group is not SumGroup.TP
|
||||
or edge.residual.sharded
|
||||
or TokenAxis.ATTN_TP_SCATTER not in edge.need.gathers_itself
|
||||
or edge.residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
|
||||
):
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
return Boundary(
|
||||
edge,
|
||||
prepare=partial(
|
||||
_attention_input_step,
|
||||
layer_input=layer_input,
|
||||
fusions=fusions,
|
||||
enters_stack=enters_stack,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
input_move=_select_attention_input_move(edge.residual_to, edge.need),
|
||||
)
|
||||
|
||||
|
||||
def _select_boundary_steps(
|
||||
sides: DecoderLayerSides,
|
||||
*,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input,
|
||||
attention_fusions: Tuple[Callable, ...] = (),
|
||||
enters_stack: bool = False,
|
||||
) -> BoundarySteps:
|
||||
"""The steps of a decoder layer: its three boundaries, each chosen from the
|
||||
declarations of its two sides, around the layer's residual operations. Both
|
||||
edges that cross attention CP take the same ``cp_moves``; the attention's
|
||||
input tries ``attention_fusions``, and the layer stack's first layer
|
||||
(``enters_stack``) starts its residual there."""
|
||||
edges = decoder_layer_edges(sides)
|
||||
out_of_ffn = make_boundary(
|
||||
edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops
|
||||
)
|
||||
into_ffn = make_boundary(
|
||||
edges.into_ffn,
|
||||
reads=InputRead.FFN,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
into_attention = make_boundary(
|
||||
edges.into_attention,
|
||||
reads=InputRead.ATTENTION,
|
||||
fusions=attention_fusions,
|
||||
residual_ops=residual_ops,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_prepare=into_attention.prepare,
|
||||
attention_input=into_attention.input_move,
|
||||
ffn_input=into_ffn.prepare,
|
||||
ffn_input_rows=into_ffn.input_rows,
|
||||
ffn_output=edges.out_of_ffn.produced,
|
||||
ffn_output_move=out_of_ffn.output_move,
|
||||
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
|
||||
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
|
||||
fused=into_ffn.fused,
|
||||
attention_handoff=attention_handoff,
|
||||
)
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
gathers them itself."""
|
||||
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
|
||||
if not need.layout.sharded <= rows.sharded or gathered not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_TP_SCATTER},
|
||||
):
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
if gathered:
|
||||
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
|
||||
def _select_ffn_input(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
residual: Layout,
|
||||
residual_to: Layout,
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
the rows it has while the FFN runs, write the output into it and read the
|
||||
FFN input, and bring the rows to what the FFN's group needs: a gather over
|
||||
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
|
||||
only when nothing is gathered or sliced, and only if it completes the sum
|
||||
the attention output owes. A write-back that is not a plain add runs only
|
||||
after the sum completes."""
|
||||
if produced.leaves_for_next_layer and produced.group not in (
|
||||
None,
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
# A producer that leaves its sum only for some batches (an FFN before
|
||||
# this one) hands that sum on with the value: complete it, then the
|
||||
# input is a complete output.
|
||||
step, fused = _select_ffn_input(
|
||||
StageOutput(produced.layout),
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=need,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(_mlp_input_completing_owed, step=step), fused
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
if owes_attention_tp != produced.always_leaves or produced.group not in (
|
||||
None,
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective; a
|
||||
# complete value is only sliced.
|
||||
if (
|
||||
sliced != {TokenAxis.ATTN_TP_SCATTER}
|
||||
or gathered
|
||||
or residual_to != need.layout
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
|
||||
scatters_residual=residual != residual_to,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# The residual stays on each rank's slice while the FFN takes the
|
||||
# attention's rows (MHC on an input-scattered batch).
|
||||
if gathered or not owes_attention_tp or residual != residual_to:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
on_chunk, fused = _select_ffn_input(
|
||||
produced,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=StageInput(produced.layout),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
or gathered
|
||||
not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_DP},
|
||||
{TokenAxis.ATTN_DP, TokenAxis.ATTN_CP},
|
||||
)
|
||||
or residual.sharded - residual_to.sharded
|
||||
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
# A residual arriving on each rank's slice is gathered back first.
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if not residual_ops.adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
fused,
|
||||
)
|
||||
# The partial order adds the residual on attention-TP rank 0 before the DP
|
||||
# gather's collective completes that sum, which only a plain residual add
|
||||
# allows.
|
||||
# Over attention DP and CP, the DP gather puts each CP rank's shard in its
|
||||
# DP group's slot, so the one DP sum gathers both axes.
|
||||
places_cp_shards = TokenAxis.ATTN_CP in gathered
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and residual_ops.adds_plainly
|
||||
):
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_dp_partial,
|
||||
gathers_residual=gathers_residual,
|
||||
places_cp_shards=places_cp_shards,
|
||||
),
|
||||
(),
|
||||
)
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
places_cp_shards=places_cp_shards,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
|
||||
|
||||
def _select_ffn_output_move(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[bool, Optional[Callable], bool]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here; and whether that move also completes the sum the FFN
|
||||
leaves."""
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, pair._trivial, False
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return False, partial(pair._gather, residual_ops=residual_ops), False
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
|
||||
# The residual stays on each rank's slice (MHC on an input-scattered
|
||||
# batch): a reduce-scatter onto the slice completes the sum the FFN
|
||||
# leaves; a complete output is only sliced.
|
||||
sums = produced.leaves_for_reduce_scatter
|
||||
return (
|
||||
False,
|
||||
partial(
|
||||
pair._onto_residual_shard,
|
||||
sums=sums,
|
||||
gathers_back=to != residual,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
sums,
|
||||
)
|
||||
if to != residual or not produced.layout.sharded <= residual.sharded:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
if returned == {TokenAxis.ATTN_CP}:
|
||||
if cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
if not produced.leaves_for_reduce_scatter:
|
||||
# A complete output: this rank's block of it, nothing summed.
|
||||
return False, cp_moves.take_back, False
|
||||
# The FFN leaves its sum: only a take-back that sums over the same
|
||||
# ranks completes it.
|
||||
if cp_moves.reduce_scatter is None or not _same_ranks(
|
||||
_sum_group(produced.group), cp_moves.reduce_scatter_group()
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return False, cp_moves.reduce_scatter, True
|
||||
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
|
||||
# This rank's CP shard, from where the DP gather put it.
|
||||
return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False
|
||||
if returned != {TokenAxis.ATTN_DP}:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return True, None, False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,251 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Token axes, layouts and process groups of what a layer hands across its
|
||||
boundaries, and the configuration and batch facts they depend on."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import Dict, FrozenSet, List, Mapping, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
dsa_use_prefill_cp,
|
||||
is_dsa_enable_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import is_mla_cp_active, is_mla_cp_enabled
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_moe_cp_size,
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
post_experts_reduction_group,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class TokenAxis(Enum):
|
||||
"""A parallel dimension across which ranks hold different tokens."""
|
||||
|
||||
ATTN_DP = auto()
|
||||
ATTN_CP = auto()
|
||||
# Each attention-TP rank holds a slice of its group's tokens.
|
||||
ATTN_TP_SCATTER = auto()
|
||||
|
||||
|
||||
class Layout(msgspec.Struct, frozen=True):
|
||||
"""The token axes a rank's rows are sharded over. Single-rank axes are left
|
||||
out, so two layouts are equal exactly when every rank holds the same tokens."""
|
||||
|
||||
sharded: FrozenSet[TokenAxis]
|
||||
|
||||
@classmethod
|
||||
def sharded_over(
|
||||
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> "Layout":
|
||||
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
|
||||
|
||||
|
||||
class SumGroup(Enum):
|
||||
"""The group a stage's output is summed over, by name: the handle comes
|
||||
from get_parallel() when the sum runs."""
|
||||
|
||||
ATTN_TP = auto()
|
||||
TP = auto()
|
||||
# The group one all-reduce of a MoE output runs over.
|
||||
MOE_OUTPUT = auto()
|
||||
|
||||
|
||||
class ScatterMode(Enum):
|
||||
"""
|
||||
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
|
||||
Model input/output: [ab, ab, cd, cd] for four ranks respectively
|
||||
SCATTERED: [a, b, c, d]
|
||||
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
|
||||
FULL: [abcd, abcd, abcd, abcd]
|
||||
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
|
||||
"""
|
||||
|
||||
SCATTERED = auto()
|
||||
TP_ATTN_FULL = auto()
|
||||
FULL = auto()
|
||||
MOE_FULL = auto()
|
||||
|
||||
@staticmethod
|
||||
def model_input_output():
|
||||
"""The scatter mode for model forward pass input and output data"""
|
||||
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
|
||||
return ScatterMode.SCATTERED
|
||||
|
||||
return ScatterMode.TP_ATTN_FULL
|
||||
|
||||
|
||||
def enable_moe_dense_fully_dp():
|
||||
return get_parallel().moe_dense_tp_size == 1
|
||||
|
||||
|
||||
def _generic_prefill_cp_shards_tokens() -> bool:
|
||||
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
|
||||
parallel = get_parallel()
|
||||
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
|
||||
|
||||
|
||||
def enable_dwdp():
|
||||
return get_parallel().dwdp_size > 1
|
||||
|
||||
|
||||
def _batch_size(forward_batch: ForwardBatch) -> int:
|
||||
return (
|
||||
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
|
||||
)
|
||||
|
||||
|
||||
def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
|
||||
if is_dp_attention_enabled():
|
||||
# The FFN runs on every DP rank's tokens, so every rank decides alike.
|
||||
return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0
|
||||
return _batch_size(forward_batch) > 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
attn_tp_rank: int
|
||||
attn_tp_size: int
|
||||
attn_dp_size: int
|
||||
attn_cp_rank: int
|
||||
attn_cp_size: int
|
||||
tp_size: int
|
||||
cache = None
|
||||
tp_rank: int
|
||||
force_layernorm_before_dp_gather: bool = False
|
||||
|
||||
@classmethod
|
||||
def init_new(cls):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_cp_size = get_parallel().attn_cp_size
|
||||
attn_cp_rank = get_parallel().attn_cp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
moe_cp_size = get_moe_cp_size()
|
||||
process_group_sizes = {
|
||||
ScatterMode.SCATTERED: 1,
|
||||
ScatterMode.TP_ATTN_FULL: attn_tp_size,
|
||||
# TODO: support --moe-dense-tp-size > 1
|
||||
# With context parallel enabled, we should exclude
|
||||
# the attn_cp_size from the total tp_size
|
||||
ScatterMode.FULL: tp_size // attn_cp_size,
|
||||
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
|
||||
}
|
||||
return cls(
|
||||
process_group_sizes=process_group_sizes,
|
||||
attn_tp_rank=attn_tp_rank,
|
||||
attn_tp_size=attn_tp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
attn_cp_rank=attn_cp_rank,
|
||||
attn_cp_size=attn_cp_size,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
|
||||
|
||||
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
|
||||
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
|
||||
gathered across the MoE-CP group, or None when it is not: only a context
|
||||
parallel extend with CP metadata gathers, and only when the group has more
|
||||
than one rank. The batch is read first, so a batch that is not a CP extend
|
||||
never reads the group."""
|
||||
if (
|
||||
forward_batch.forward_mode.is_context_parallel_extend()
|
||||
and forward_batch.attn_cp_metadata is not None
|
||||
and get_moe_cp_size() > 1
|
||||
):
|
||||
return forward_batch.attn_cp_metadata.per_rank_actual_token
|
||||
return None
|
||||
|
||||
|
||||
def sparse_moe_gathers_over_moe_cp() -> bool:
|
||||
"""Whether a sparse MoE's input is gathered over the MoE-CP group on a CP
|
||||
extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than
|
||||
the MoE's data-parallel groups. DSA and MLA CP gather over attention CP
|
||||
instead."""
|
||||
return (
|
||||
not is_moe_input_scattered_across_dp_ranks()
|
||||
and is_enable_moe_cp_allgather()
|
||||
and not _gathers_over_attention_cp()
|
||||
)
|
||||
|
||||
|
||||
def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether a sparse MoE's input is gathered over the MoE-CP group on this
|
||||
batch: a CP extend, for a MoE that gathers there."""
|
||||
return (
|
||||
sparse_moe_gathers_over_moe_cp()
|
||||
and moe_cp_gathered_rows(forward_batch) is not None
|
||||
)
|
||||
|
||||
|
||||
def _cp_shard_token_rows(forward_batch: ForwardBatch) -> List[int]:
|
||||
"""Rows of each CP rank's shard that hold tokens; the shards are padded to
|
||||
one length after them."""
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
return metadata.per_rank_logical_token or metadata.per_rank_actual_token
|
||||
|
||||
|
||||
def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if group is SumGroup.ATTN_TP:
|
||||
return parallel.attn_tp_group
|
||||
if group is SumGroup.TP:
|
||||
return parallel.tp_group
|
||||
return post_experts_reduction_group()
|
||||
|
||||
|
||||
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
|
||||
"""The token axes' sizes for a batch: attention CP shards tokens only on
|
||||
a CP extend (``cp_active``); otherwise every CP rank holds them all."""
|
||||
parallel = get_parallel()
|
||||
return {
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
}
|
||||
|
||||
|
||||
def _gathers_over_attention_cp() -> bool:
|
||||
"""Whether a CP extend gathers the FFN input over the attention-CP group in
|
||||
equal shards and takes the output back with a reduce-scatter there: DSA and
|
||||
MLA CP. GQA prefill CP gathers over the MoE-CP group instead."""
|
||||
return is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
||||
|
||||
|
||||
def _batch_shards_over_cp(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether this batch's tokens are split across the CP ranks. Only a context
|
||||
parallel extend is, so other batches, decode graph capture among them,
|
||||
never read the CP predicates."""
|
||||
if not forward_batch.forward_mode.is_context_parallel_extend():
|
||||
return False
|
||||
if _gathers_over_attention_cp():
|
||||
return dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch)
|
||||
return moe_cp_gathered_rows(forward_batch) is not None
|
||||
|
||||
|
||||
def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool:
|
||||
return sorted(a.ranks) == sorted(b.ranks)
|
||||
@@ -0,0 +1,794 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""The communication a boundary runs: the attention input, the FFN input and the
|
||||
FFN output's way back, over attention TP, DP and CP."""
|
||||
|
||||
from typing import Callable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
GroupCoordinator,
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
attention_tensor_model_parallel_quant_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output
|
||||
from sglang.srt.layers.communicator.residual import ResidualOps
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
can_use_dp_reduce_scatter,
|
||||
dp_gather_partial,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
dp_scatter,
|
||||
get_dp_global_num_tokens,
|
||||
get_global_dp_buffer,
|
||||
get_local_dp_buffer,
|
||||
get_moe_cp_rank,
|
||||
get_moe_cp_size,
|
||||
is_allocation_symmetric,
|
||||
moe_cp_all_gather_into_tensor,
|
||||
)
|
||||
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
is_npu,
|
||||
)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
||||
|
||||
|
||||
def tp_reduce_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
context: "CommunicateContext",
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Module-level so MHC communicators can reuse it without a
|
||||
``LayerCommunicator`` instance."""
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
assert hidden_states.shape[0] % context.tp_size == 0, (
|
||||
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
|
||||
)
|
||||
local_tokens = hidden_states.shape[0] // context.tp_size
|
||||
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
|
||||
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
|
||||
if residual is not None:
|
||||
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
||||
return output, residual
|
||||
|
||||
|
||||
def layer_input_buffer(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
) -> torch.Tensor:
|
||||
"""The tensor holding a layer's input, for reusing its memory without reading it."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return hidden_states.partial
|
||||
return hidden_states
|
||||
|
||||
|
||||
class CommunicateSimpleFn:
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
@staticmethod
|
||||
def _scattered_to_tp_attn_full(
|
||||
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
if isinstance(hidden_states, tuple):
|
||||
gathered_hidden_states = []
|
||||
for local_hidden_states in hidden_states:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
output = torch.empty(
|
||||
(
|
||||
local_hidden_states.shape[0] * context.attn_tp_size,
|
||||
*local_hidden_states.shape[1:],
|
||||
),
|
||||
dtype=local_hidden_states.dtype,
|
||||
device=local_hidden_states.device,
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(
|
||||
output,
|
||||
local_hidden_states,
|
||||
)
|
||||
gathered_hidden_states.append(output)
|
||||
return tuple(gathered_hidden_states)
|
||||
|
||||
return _redistribute_from_attn_tp_shards(hidden_states)
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states: torch.Tensor, context: CommunicateContext
|
||||
) -> torch.Tensor:
|
||||
local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
|
||||
return local_hidden_states
|
||||
|
||||
|
||||
def _redistribute_input_to_moe_cp(
|
||||
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
|
||||
) -> torch.Tensor:
|
||||
# Zigzag split can produce unequal token counts across CP ranks
|
||||
# (when seq_len % (cp_size * 2) != 0). NCCL allgather requires
|
||||
# equal input sizes, so pad to the max per-rank token count.
|
||||
max_tokens = max(rows)
|
||||
pad_size = max_tokens - hidden_states.shape[0]
|
||||
if pad_size > 0:
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size])
|
||||
|
||||
output = torch.empty(
|
||||
(max_tokens * moe_cp_size, hidden_states.shape[1]),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
moe_cp_all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
def _mlp_input_reduce_output(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
may_quantize
|
||||
and not forward_batch.forward_mode.is_decode_or_idle()
|
||||
and get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
|
||||
return attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
|
||||
def _redistribute_input_to_dp(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
cp_shard_counts: Optional[List[int]] = None,
|
||||
) -> torch.Tensor:
|
||||
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
|
||||
dp_gather_replicate(
|
||||
global_hidden_states, hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
return global_hidden_states
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_to_dp(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
cp_shard_counts: Optional[List[int]] = None,
|
||||
) -> torch.Tensor:
|
||||
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
|
||||
dp_gather_partial(
|
||||
global_hidden_states, hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
return global_hidden_states
|
||||
|
||||
|
||||
def move_rows(
|
||||
hidden_states: torch.Tensor,
|
||||
rows: Layout,
|
||||
to: Layout,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
"""A complete value from the rows it is on to ``to``: gathered over the
|
||||
token axes ``to`` does not shard (attention TP, then attention DP), or cut to
|
||||
this rank's share of those it does (attention DP, then attention TP)."""
|
||||
gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded
|
||||
if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut:
|
||||
raise NotImplementedError(f"{rows=} {to=}")
|
||||
if TokenAxis.ATTN_TP_SCATTER in gathered:
|
||||
hidden_states = _redistribute_from_attn_tp_shards(hidden_states)
|
||||
if TokenAxis.ATTN_DP in gathered:
|
||||
hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch)
|
||||
if TokenAxis.ATTN_DP in cut:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
if TokenAxis.ATTN_TP_SCATTER in cut:
|
||||
parallel = get_parallel()
|
||||
hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
|
||||
parallel.attn_tp_rank
|
||||
]
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _mlp_input_without_dp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
if gathers_residual:
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
|
||||
)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
|
||||
residual,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_dp_partial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
):
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then
|
||||
normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP
|
||||
group's tokens beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
if context.attn_tp_rank == 0:
|
||||
hidden_states += residual
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
hidden_states = _reduce_and_redistribute_output_to_dp(
|
||||
hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = layernorm(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _hand_qkv_hook_its_input(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Give the attention's QKV hook the attention input."""
|
||||
if qkv_latent_func is not None:
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _hand_scattered_input_to_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Input-scattered attention takes each rank's slice, and its QKV hook
|
||||
gathers the rows after the projection. DSA and attention without a hook
|
||||
consume full hidden states, so those are gathered here, and the hook is
|
||||
told they are."""
|
||||
ctx = get_attn_tp_context()
|
||||
if ctx.is_dsa or qkv_latent_func is None:
|
||||
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
|
||||
if qkv_latent_func is not None:
|
||||
ctx.set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
qkv_latent_func,
|
||||
is_pre_gathered=ctx.is_dsa,
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _tp_all_gather_scattered_rows(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
# Input-scattered attention keeps the same number of tokens on every TP rank.
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
def _mlp_input_completing_owed(
|
||||
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
step: Callable,
|
||||
):
|
||||
"""An FFN input whose producer may have left its sum: complete what the
|
||||
value owes, then run ``step`` on the complete output."""
|
||||
return step(
|
||||
reduce_output(hidden_states), residual, forward_batch, layernorm, context
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_norm(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states, context
|
||||
)
|
||||
if scatters_residual:
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_slice(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_residual_into_sum(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context
|
||||
)
|
||||
|
||||
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
scattered_states += residual
|
||||
residual = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = layernorm(residual)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_gather_attention_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gather: Callable,
|
||||
):
|
||||
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context
|
||||
)
|
||||
return dsa_cp_gather_hidden_states(hidden_states), residual
|
||||
|
||||
|
||||
def _mlp_input_gather_moe_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gather: Callable,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
||||
TP_ATTN_FULL."""
|
||||
# Early return on empty tensor is safe for MOE_CP because:
|
||||
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
||||
# so no rank hits this path while others proceed to the allgather.
|
||||
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
||||
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None and hidden_states.shape[0] > 0:
|
||||
hidden_states = _redistribute_input_to_moe_cp(
|
||||
hidden_states, rows, get_moe_cp_size()
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _dp_scatter_group() -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if parallel.tp_size == parallel.attn_dp_size:
|
||||
return parallel.tp_group
|
||||
return parallel.attn_tp_group
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_varlen(
|
||||
local_hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> None:
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
hidden_states,
|
||||
output=local_hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_max_len(
|
||||
local_hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> None:
|
||||
dp_reduce_scatter_tensor(local_hidden_states, hidden_states)
|
||||
|
||||
|
||||
def _redistribute_output(
|
||||
local_hidden_states: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> None:
|
||||
dp_scatter(local_hidden_states, hidden_states, forward_batch)
|
||||
|
||||
|
||||
def _redistribute_output_from_moe_cp(
|
||||
hidden_states: torch.Tensor, rows: List[int]
|
||||
) -> torch.Tensor:
|
||||
moe_cp_rank = get_moe_cp_rank()
|
||||
# The allgather was padded to max_tokens_per_rank (equal chunks).
|
||||
# Extract this rank's actual (non-padded) tokens from its chunk.
|
||||
max_tokens_per_rank = max(rows)
|
||||
actual_local_tokens = rows[moe_cp_rank]
|
||||
return hidden_states.narrow(
|
||||
0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_step(
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings an FFN output gathered over attention
|
||||
DP back to this rank's tokens when the FFN leaves its sum to it (see
|
||||
StageOutput); None when the FFN reduces the output and only a scatter
|
||||
remains."""
|
||||
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
|
||||
return _reduce_and_redistribute_output_varlen
|
||||
if (
|
||||
leaves_for_reduce_scatter
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and can_use_dp_reduce_scatter()
|
||||
):
|
||||
return _reduce_and_redistribute_output_max_len
|
||||
return None
|
||||
|
||||
|
||||
def _to_local_tokens(
|
||||
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
|
||||
forward_batch: ForwardBatch,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
|
||||
step(local_hidden_states, hidden_states, forward_batch)
|
||||
return local_hidden_states
|
||||
|
||||
|
||||
def _all_reduce_then_to_local_tokens(
|
||||
group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return _to_local_tokens(
|
||||
_redistribute_output, forward_batch, group.all_reduce(hidden_states)
|
||||
)
|
||||
|
||||
|
||||
class CommunicateSummableTensorPairFn:
|
||||
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
is_layer_sparse: bool = False,
|
||||
):
|
||||
step = (
|
||||
_reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=allow_reduce_scatter,
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever it
|
||||
# applies (should_skip_post_experts_all_reduce).
|
||||
leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
|
||||
)
|
||||
or _redistribute_output
|
||||
)
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
|
||||
@staticmethod
|
||||
def _gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = residual_ops.update_residual(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _onto_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
residual_ops: ResidualOps,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = residual_ops.update_residual(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
assert residual is None, "not yet handled residual!=None"
|
||||
return _redistribute_to_attn_tp_shards(hidden_states, context), None
|
||||
|
||||
@staticmethod
|
||||
def _take_back_attention_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: this rank's shard of a complete output gathered in
|
||||
equal shards over the attention-CP group."""
|
||||
parallel = get_parallel()
|
||||
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
|
||||
return shard, residual
|
||||
|
||||
@staticmethod
|
||||
def _reduce_scatter_over_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
keep this rank's shard."""
|
||||
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
|
||||
|
||||
@staticmethod
|
||||
def _take_back_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""This rank's CP shard of the rows the DP gather put in its DP group's
|
||||
slot, at the shard's padded length with the padding zeroed."""
|
||||
held = forward_batch.attn_cp_metadata.per_rank_actual_token
|
||||
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
|
||||
: held[get_parallel().attn_cp_rank]
|
||||
]
|
||||
dp_scatter(
|
||||
local_hidden_states,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
_cp_shard_token_rows(forward_batch),
|
||||
)
|
||||
return local_hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
||||
|
||||
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
||||
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
||||
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
||||
CP-local portion.
|
||||
|
||||
If DP>1, further scatter back to the local DP slice.
|
||||
"""
|
||||
# Only scatter back during prefill; decode was never allgathered so no-op.
|
||||
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
|
||||
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None:
|
||||
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
|
||||
|
||||
if context.attn_dp_size > 1:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
@@ -0,0 +1,59 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Outputs that carry work their consumer still owes."""
|
||||
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its sum, left for the next layer's input.
|
||||
Hand it to the next layer, or pass it through reduce_output() before reading
|
||||
it any other way. The producer says what is owed: one all-reduce over
|
||||
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
|
||||
|
||||
partial: torch.Tensor
|
||||
group: Optional[GroupCoordinator] = None
|
||||
# Under attention DP: the reduction that also brings ``partial`` back to this
|
||||
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
|
||||
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
|
||||
|
||||
class HandoffOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes work only its producer knows how to do (a
|
||||
MoE's finalize and sum), left for the next layer's input. A fused kernel
|
||||
there may do that work together with its own; anything else passes it
|
||||
through reduce_output(), which calls ``complete()``."""
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
"""Do the owed work, unfused, and return the complete output."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def reduce_output(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None],
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Run the work an UnreducedOutput or a HandoffOutput still owes; pass
|
||||
anything else through."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
if hidden_states.reduce_and_redistribute is not None:
|
||||
return hidden_states.reduce_and_redistribute(hidden_states.partial)
|
||||
return hidden_states.group.all_reduce(hidden_states.partial)
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states.complete()
|
||||
return hidden_states
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""How a layer writes its stages' outputs into the residual and reads the next
|
||||
stage's input back."""
|
||||
|
||||
from typing import Protocol, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class ResidualOps(Protocol):
|
||||
"""How a layer writes each stage's output into its residual and reads the
|
||||
next stage's input from it. The boundary steps complete sums and move
|
||||
tokens around these operations. AddAndNorm adds and normalizes; MHC's
|
||||
hyper-connection streams implement the same operations (residual.mhc)."""
|
||||
|
||||
# The write-back is a plain add, which one rank may run before the sum it
|
||||
# adds into completes: the DP partial order, and a residual that joins the
|
||||
# attention output's sum.
|
||||
adds_plainly: bool
|
||||
# The layer writes its FFN output into the residual itself instead of
|
||||
# leaving that to the next layer's input.
|
||||
updates_residual_after_ffn: bool
|
||||
|
||||
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
|
||||
"""The attention input and the residual, from a residual that already
|
||||
holds the previous layer's output."""
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
|
||||
) -> Tuple:
|
||||
"""Write the previous layer's output into the residual, then read the
|
||||
attention input from it."""
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
|
||||
"""Write the attention output into the residual, then read the FFN
|
||||
input from it."""
|
||||
|
||||
def update_residual(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the FFN output written into it."""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual."""
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
|
||||
"""The residual gathered from every attention-TP rank's slice."""
|
||||
@@ -0,0 +1,311 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""The plain residual: add and RMSNorm, and the fused all-reduce + add + norm
|
||||
kernels."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
is_flashinfer_allreduce_unavailable,
|
||||
uses_cutedsl_ar_fusion,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
materialize_bpreshuffle_fp8_scale_tuple,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_platform
|
||||
from sglang.srt.utils import (
|
||||
get_bool_env_var,
|
||||
is_cuda,
|
||||
is_flashinfer_available,
|
||||
is_gfx95_supported,
|
||||
is_gfx1250_supported,
|
||||
is_hip,
|
||||
)
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_flashinfer_available = is_flashinfer_available()
|
||||
_is_sm90_supported = _is_cuda and get_platform().is_sm90
|
||||
_is_sm100_supported = _is_cuda and get_platform().is_sm100
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
_is_gfx1250_supported = is_gfx1250_supported()
|
||||
|
||||
|
||||
if _use_aiter:
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype
|
||||
|
||||
if _is_gfx1250_supported:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
||||
else:
|
||||
from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant
|
||||
from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant
|
||||
|
||||
if _is_gfx95_supported:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
||||
|
||||
from sglang.srt.layers.quantization.rocm_mxfp4_utils import (
|
||||
fused_rms_mxfp4_quant,
|
||||
)
|
||||
|
||||
|
||||
def _fused_rmsnorm_fp8_per_token_quant(
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Fused (optional residual-add +) RMSNorm + FP8 per-token quantization.
|
||||
|
||||
Only used with the aiter (ROCm) backend.
|
||||
|
||||
Args:
|
||||
residual: if provided, computes hidden_states + residual before RMSNorm
|
||||
and returns updated residual_out as second element.
|
||||
|
||||
Returns:
|
||||
If residual is None: (out_fp8, scale)
|
||||
If residual provided: ((out_fp8, scale), residual_out)
|
||||
"""
|
||||
if _is_gfx1250_supported:
|
||||
# per-token quant == group quant with group_size == hidden size, giving
|
||||
# an (M, 1) scale.
|
||||
N = hidden_states.shape[-1]
|
||||
(out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant(
|
||||
hidden_states,
|
||||
weight,
|
||||
epsilon,
|
||||
group_size=N,
|
||||
dtype_quant=_aiter_fp8_dtype,
|
||||
res1=residual,
|
||||
)
|
||||
if residual is not None:
|
||||
return (out_fp8, scale), residual_out
|
||||
return (out_fp8, scale)
|
||||
|
||||
M, N = hidden_states.shape
|
||||
out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device)
|
||||
scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device)
|
||||
if residual is not None:
|
||||
residual_out = torch.empty_like(hidden_states)
|
||||
_aiter_add_rmsnorm_quant(
|
||||
out_fp8,
|
||||
hidden_states,
|
||||
residual,
|
||||
residual_out,
|
||||
scale,
|
||||
weight,
|
||||
epsilon,
|
||||
0, # group_size=0 → per-token
|
||||
)
|
||||
return (out_fp8, scale.unsqueeze(1)), residual_out
|
||||
else:
|
||||
_aiter_rmsnorm_quant(
|
||||
out_fp8,
|
||||
hidden_states,
|
||||
scale,
|
||||
weight,
|
||||
epsilon,
|
||||
0, # group_size=0 → per-token
|
||||
)
|
||||
return (out_fp8, scale.unsqueeze(1))
|
||||
|
||||
|
||||
# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465
|
||||
# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True).
|
||||
FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048
|
||||
|
||||
|
||||
def apply_flashinfer_allreduce_fusion(batch_size: int):
|
||||
return (
|
||||
# NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion
|
||||
# Ref: https://github.com/sgl-project/sglang/issues/17237
|
||||
(_is_sm90_supported or _is_sm100_supported)
|
||||
and _is_flashinfer_available
|
||||
and not is_dp_attention_enabled()
|
||||
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
||||
# cutedsl runs its own fused path from the fusion communicator.
|
||||
and not uses_cutedsl_ar_fusion()
|
||||
and not is_flashinfer_allreduce_unavailable()
|
||||
# Symbolic size checks stay last: under Dynamo tracing they guard on
|
||||
# the dynamic token dim, so statically-off configs must short-circuit
|
||||
# before reaching them.
|
||||
and batch_size > 0
|
||||
and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
||||
)
|
||||
|
||||
|
||||
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
|
||||
comm = get_exec().comm
|
||||
if not comm.enable_aiter_allreduce_fusion:
|
||||
return False
|
||||
if forward_mode.is_extend_or_draft_extend_or_mixed():
|
||||
return not comm.disable_aiter_allreduce_fusion_in_prefill
|
||||
return not comm.disable_aiter_allreduce_fusion_in_decode
|
||||
|
||||
|
||||
def apply_aiter_all_reduce_fusion(
|
||||
input_tensor: torch.Tensor, forward_batch: ForwardBatch
|
||||
):
|
||||
n = input_tensor.shape[-1]
|
||||
total_bytes = input_tensor.numel() * input_tensor.element_size()
|
||||
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
|
||||
return (
|
||||
_use_aiter
|
||||
and total_bytes > 0
|
||||
and n <= 16384
|
||||
and total_bytes <= 8 * 1024 * 8192
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
|
||||
|
||||
def _update_and_read_residual_plain(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
):
|
||||
if residual is None:
|
||||
return norm(hidden_states), hidden_states
|
||||
return norm(hidden_states, residual, post_residual_addition)
|
||||
|
||||
|
||||
def _update_and_read_residual_aiter_mxfp4(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
):
|
||||
# post_residual_addition is not applied on this path.
|
||||
output, *_, residual_out = fused_rms_mxfp4_quant(
|
||||
hidden_states,
|
||||
norm.weight,
|
||||
norm.variance_epsilon,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
residual,
|
||||
)
|
||||
return output, hidden_states if residual is None else residual_out
|
||||
|
||||
|
||||
def _update_and_read_residual_aiter_fp8_group(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
):
|
||||
"""aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the
|
||||
unquantized bf16 output rides along as a third element, so the DSA indexer
|
||||
can skip dequantizing. post_residual_addition is not applied on this path."""
|
||||
needs_bf16 = get_attn_tp_context().is_dsa
|
||||
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
|
||||
hidden_states,
|
||||
norm.weight,
|
||||
norm.variance_epsilon,
|
||||
inp2=None,
|
||||
inp2_weight=None,
|
||||
inp2_epsilon=None,
|
||||
group_size=128,
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=residual,
|
||||
output_unquantized_inp1=needs_bf16,
|
||||
transpose_scale=False,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
output = materialize_bpreshuffle_fp8_scale_tuple(output)
|
||||
if needs_bf16:
|
||||
output = (output[0], output[1], unquantized)
|
||||
return output, hidden_states if residual is None else residual_out
|
||||
|
||||
|
||||
def _update_and_read_residual_aiter_fp8_per_token(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
):
|
||||
if residual is None:
|
||||
output = _fused_rmsnorm_fp8_per_token_quant(
|
||||
hidden_states, norm.weight.data, norm.variance_epsilon
|
||||
)
|
||||
return output, hidden_states
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
return _fused_rmsnorm_fp8_per_token_quant(
|
||||
hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual
|
||||
)
|
||||
|
||||
|
||||
def _attn_input_update_and_read_residual(quant_format: str):
|
||||
"""Add the previous layer's output to the residual and read the attention
|
||||
input from it: the input norm, fused with the quantization this format
|
||||
wants. Without a residual (the first layer, or one already folded into
|
||||
hidden_states) the input itself becomes the residual."""
|
||||
if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format:
|
||||
return _update_and_read_residual_aiter_mxfp4
|
||||
if _use_aiter and _is_gfx95_supported and quant_format == "fp8":
|
||||
return _update_and_read_residual_aiter_fp8_group
|
||||
if _use_aiter and quant_format == "fp8_per_token":
|
||||
return _update_and_read_residual_aiter_fp8_per_token
|
||||
return _update_and_read_residual_plain
|
||||
|
||||
|
||||
class AddAndNorm:
|
||||
"""A plain residual: a stage's output is added into it, and the next
|
||||
stage's input is its norm. An empty batch skips the norms."""
|
||||
|
||||
adds_plainly = True
|
||||
updates_residual_after_ffn = False
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, residual, None, None
|
||||
)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
if residual is None:
|
||||
# The layer stack starts at this FFN: its input is the residual.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return norm(hidden_states), hidden_states
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return _redistribute_to_attn_tp_shards(residual, context)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
ADD_AND_NORM = AddAndNorm()
|
||||
+2
-70
@@ -11,19 +11,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Hyper-connection residual streams."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis
|
||||
from sglang.srt.layers.communicator import (
|
||||
BoundarySteps,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -141,66 +136,3 @@ class MHCState:
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
|
||||
|
||||
class MHCLayerCommunicator(LayerCommunicator):
|
||||
"""A layer whose residual is hyper-connection streams: the shared boundary
|
||||
steps, run with MHCState's residual operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_scatter_modes: LayerScatterModes,
|
||||
input_layernorm: torch.nn.Module,
|
||||
post_attention_layernorm: torch.nn.Module,
|
||||
allow_reduce_scatter: bool = False,
|
||||
qkv_latent_func: Optional[Callable] = None,
|
||||
*,
|
||||
hc_mult: int,
|
||||
hc_attn_pre: Callable,
|
||||
hc_ffn_pre: Callable,
|
||||
hc_post: Callable,
|
||||
hc_ffn_post_pre: Optional[Callable] = None,
|
||||
):
|
||||
self.mhc = MHCState(
|
||||
hc_mult=hc_mult,
|
||||
hc_attn_pre=hc_attn_pre,
|
||||
hc_ffn_pre=hc_ffn_pre,
|
||||
hc_post=hc_post,
|
||||
hc_ffn_post_pre=hc_ffn_post_pre,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support a MoE gathered over the "
|
||||
"MoE-CP group (moe_dp_size < attention_context_parallel_size). "
|
||||
"Increase moe_dp_size to match attention_context_parallel_size."
|
||||
)
|
||||
# The postprocess writes the FFN output into the streams, so the FFN's
|
||||
# sum never waits for the next layer.
|
||||
super().__init__(
|
||||
layer_scatter_modes,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
allow_reduce_scatter,
|
||||
qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
residual_ops=self.mhc,
|
||||
)
|
||||
# MHC has not been run with input-scattered attention under attention CP.
|
||||
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator with input-scattered attention under attention CP"
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
) -> BoundarySteps:
|
||||
# MHC has not been run with an FFN input gathered over attention CP.
|
||||
if (
|
||||
TokenAxis.ATTN_CP
|
||||
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
|
||||
)
|
||||
return super()._steps_from_declarations(sides, **kwargs)
|
||||
@@ -12,12 +12,13 @@ from typing import Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis
|
||||
from sglang.srt.layers.communicator import (
|
||||
FfnExitFusion,
|
||||
FusedMlpInput,
|
||||
HandoffOutput,
|
||||
LayerCommunicator,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
UnreducedOutput,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
|
||||
@@ -28,12 +28,12 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
MHCLayerCommunicator,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelBatchedLinear,
|
||||
|
||||
@@ -3,21 +3,19 @@
|
||||
from typing import Optional
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
from sglang.srt.layers.communicator import (
|
||||
InputRead,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerStage,
|
||||
Layout,
|
||||
ScatterMode,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
stage_edges,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
InputRead,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
LayerStage,
|
||||
ScatterMode,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Patching a name the communicator package's modules look up.
|
||||
|
||||
Each module of ``sglang.srt.layers.communicator`` binds the names it imports
|
||||
in its own namespace, so a test double has to go into every module that reads
|
||||
the name, not only into the package.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.layers import communicator
|
||||
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
|
||||
from sglang.srt.layers.communicator import residual as residual_ops
|
||||
from sglang.srt.layers.communicator.adapters import attention
|
||||
from sglang.srt.layers.communicator.residual import add_norm, mhc
|
||||
|
||||
COMMUNICATOR_MODULES = (
|
||||
communicator,
|
||||
layout,
|
||||
output,
|
||||
attention,
|
||||
residual_ops,
|
||||
add_norm,
|
||||
mhc,
|
||||
ops,
|
||||
boundary,
|
||||
layer,
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def patch_communicator(name, new=mock.DEFAULT, **kwargs):
|
||||
"""``mock.patch.object(module, name, new, **kwargs)`` on every communicator
|
||||
module that has ``name``, all with the same replacement; with
|
||||
``create=True`` and no module having it, on every module. Yields the
|
||||
replacement."""
|
||||
targets = [m for m in COMMUNICATOR_MODULES if hasattr(m, name)]
|
||||
if not targets:
|
||||
if not kwargs.get("create"):
|
||||
raise AttributeError(f"no communicator module has {name!r}")
|
||||
targets = list(COMMUNICATOR_MODULES)
|
||||
with contextlib.ExitStack() as stack:
|
||||
replacement = stack.enter_context(
|
||||
mock.patch.object(targets[0], name, new, **kwargs)
|
||||
)
|
||||
for module in targets[1:]:
|
||||
stack.enter_context(
|
||||
mock.patch.object(
|
||||
module, name, replacement, create=kwargs.get("create", False)
|
||||
)
|
||||
)
|
||||
yield replacement
|
||||
Reference in New Issue
Block a user