[Refactor] Split the layer communicator into a package (move only) (#41439)

This commit is contained in:
Cheng Wan
2026-09-27 16:54:41 -07:00
committed by GitHub
parent 427c9e05c1
commit 0e907b755d
40 changed files with 4530 additions and 4039 deletions
@@ -14,9 +14,9 @@ from sglang.srt.batch_overlap.operations import (
)
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.boundary_layout import Layout
from sglang.srt.layers.communicator import (
CommunicateContext,
Layout,
reduce_output,
tbo_split_moves,
)
-403
View File
@@ -1,403 +0,0 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Token layouts of the tensors handed across layer communication boundaries."""
from enum import Enum, auto
from typing import FrozenSet, Mapping, Optional, Tuple
import msgspec
class TokenAxis(Enum):
"""A parallel dimension across which ranks hold different tokens."""
ATTN_DP = auto()
ATTN_CP = auto()
# Each attention-TP rank holds a slice of its group's tokens.
ATTN_TP_SCATTER = auto()
class Layout(msgspec.Struct, frozen=True):
"""The token axes a rank's rows are sharded over. Single-rank axes are left
out, so two layouts are equal exactly when every rank holds the same tokens."""
sharded: FrozenSet[TokenAxis]
@classmethod
def sharded_over(
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
) -> "Layout":
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
class SumGroup(Enum):
"""The group a stage's output is summed over, by name: the handle comes
from get_parallel() when the sum runs."""
ATTN_TP = auto()
TP = auto()
# The group one all-reduce of a MoE output runs over.
MOE_OUTPUT = auto()
class StageInput(msgspec.Struct, frozen=True):
"""The rows a stage's consumer needs: sharded over the token axes its
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
class StageOutput(msgspec.Struct, frozen=True):
"""What a stage's producer hands the boundary after it, fixed at
construction: its rows, the group its output is summed over, and when it
leaves that sum to the boundary instead of completing it.
What one output actually owes is carried with the value: a plain
tensor handed across layers is complete, an ``UnreducedOutput`` names what
is left. A raw compute output entering a boundary is read with this
declaration and the boundary's decision for the batch."""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The producer never reduces its output, e.g. a row-parallel projection
# built with reduce_results=False.
always_leaves: bool = False
# Otherwise it leaves the sum only when the boundary publishes the flag for
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
leaves_for_next_layer: bool = False
leaves_for_reduce_scatter: bool = False
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
class DecoderLayerSides(msgspec.Struct, frozen=True):
"""What a decoder layer declares about its attention, its FFN and the rows
between them; ``decoder_layer_edges`` turns it into the layer's
boundaries."""
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageInput
attention_output: StageOutput
ffn: StageInput
ffn_output: StageOutput
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
# The group the layer's input arrives as a partial sum over, if any.
input_owes: Optional[SumGroup] = None
# Whether the residual is added into one rank's share of the attention
# output's sum before that sum completes, instead of after it.
residual_joins_attention_sum: bool = False
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
rows the residual is on before and after the boundary."""
produced: StageOutput
need: StageInput
residual: Layout
residual_to: Layout
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
class DecoderLayerEdges(msgspec.Struct, frozen=True):
"""A decoder layer's three boundaries: from the rows the previous layer
handed on into the attention, from the attention into the FFN, and from
the FFN onto the rows the layer hands on. The next layer runs the rest of
the last one."""
into_attention: EdgeDecl
into_ffn: EdgeDecl
out_of_ffn: EdgeDecl
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
"""The boundaries of a decoder layer, each with the declarations of its two
sides. What arrives at the layer is complete unless it owes a sum by
construction (``input_owes``); a sum the previous layer leaves for a batch
comes with the value."""
owes = sides.input_owes is not None
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
attention_rows = (
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
if owes
else sides.input_rows
)
return DecoderLayerEdges(
into_attention=EdgeDecl(
produced=arrived,
need=sides.attention,
residual=sides.input_rows,
residual_to=attention_rows,
),
into_ffn=EdgeDecl(
produced=sides.attention_output,
need=sides.ffn,
residual=attention_rows,
residual_to=sides.ffn_residual_rows,
residual_joins_sum=sides.residual_joins_attention_sum,
),
out_of_ffn=EdgeDecl(
produced=sides.ffn_output,
need=StageInput(sides.output_rows),
residual=sides.ffn_residual_rows,
residual_to=sides.output_rows,
),
)
def stage_edges(
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
) -> Tuple[EdgeDecl, EdgeDecl]:
"""The two boundaries of a layer that is one stage of a sequence of stages:
into it from the previous stage's output (None at the start of the layer
stack) and out of it onto ``rows``, the rows every layer hands on and the
residual is on between stages. The previous output arrives on those rows: a
stage whose output is elsewhere (an FFN on the TP group) moves it back there
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
EdgeDecl(
produced=stage.output,
need=StageInput(rows),
residual=during,
residual_to=rows,
),
)
def sequence_parallel_layer_sides(
*, axis_sizes: Mapping[TokenAxis, int]
) -> DecoderLayerSides:
"""A decoder layer while a LayerNorm SP region is active. Its linears
all-gather their input and reduce-scatter their output themselves, so the
layer's rows, its residual and both outputs stay on each TP rank's slice."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather the slices into their GEMM.
attention=StageInput(attention, gathers_itself=gathers),
# o_proj and down reduce-scatter out of theirs, whether fused or not.
attention_output=StageOutput(local),
ffn=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
),
ffn_output=StageOutput(local),
ffn_residual_rows=local,
output_rows=local,
)
def scattered_residual_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
is_first_layer: bool,
is_last_layer: bool,
leaves_for_reduce_scatter: bool,
) -> DecoderLayerSides:
"""A decoder layer on an input-scattered batch whose residual stays on
each attention-TP rank's slice (MHC). The attention and the FFN compute on
the full rows. The attention takes the slice and gathers it itself (its
QKV hook, or the handoff before an attention that needs the rows); the
boundary brings the FFN its rows and moves both outputs back onto the
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
is the embedding's partial sum on the full rows; the last layer hands on
the full rows again."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
ffn_output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
),
ffn_residual_rows=local,
output_rows=attention if is_last_layer else local,
input_owes=SumGroup.TP if is_first_layer else None,
)
def input_scattered_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
hands_on_partial: bool,
) -> DecoderLayerSides:
"""A decoder layer on a batch whose attention input is scattered over
attention TP. Each layer's input is a partial sum over TP on the full rows:
the embedding and every FFN before it leave that sum, and a reduce-scatter
completes it onto each rank's slice, which the attention gathers itself.
The residual comes back to the full rows inside the attention output's sum."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
return DecoderLayerSides(
input_rows=attention,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next layer's input completes the sum; the last layer's FFN does.
ffn_output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
),
ffn_residual_rows=attention,
output_rows=attention,
input_owes=SumGroup.TP,
residual_joins_attention_sum=True,
)
def decoder_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_on_local_rows: bool,
previous_on_local_rows: bool,
is_last_layer: bool,
attention_gathers_local_rows: bool,
ffn_group: SumGroup,
leaves_for_next_layer: bool,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
hands_on_attention_rows: bool = False,
ffn_shards_over_cp: bool = False,
) -> DecoderLayerSides:
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine, or a dense MLP on every rank).
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
computes each CP shard on its own ranks: its rows stay sharded over CP.
A layer on local rows hands on the attention's rows when it is the last, or
with ``hands_on_attention_rows``."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
# Each attention-TP rank's own slice of those rows.
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = (
local
if ffn_on_local_rows
else Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
axis_sizes=axis_sizes,
)
)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
input_rows=local if previous_on_local_rows else attention,
attention=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
attention,
group=SumGroup.ATTN_TP if attention_tp else None,
always_leaves=attention_tp,
),
ffn=StageInput(ffn),
ffn_output=(
StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
group=ffn_group,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
)
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=(
local
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
else attention
),
)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,124 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Communication between a decoder layer's stages."""
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
DecoderLayerSides,
EdgeDecl,
FusedMlpInput,
InputRead,
LayerStage,
StageDecl,
StageInput,
StageOutput,
decoder_layer_edges,
decoder_layer_sides,
input_scattered_layer_sides,
make_boundary,
sequence_parallel_layer_sides,
stage_edges,
tbo_split_moves,
)
from sglang.srt.layers.communicator.layer import (
FfnCompletion,
FfnExit,
FfnExitFusion,
LayerCommunicator,
LayerScatterModes,
MHCLayerCommunicator,
MixerExit,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
ScatterMode,
SumGroup,
TokenAxis,
enable_moe_dense_fully_dp,
moe_cp_gathers_sparse_moe_input,
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
layer_input_buffer,
move_rows,
tp_reduce_scatter,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual.add_norm import (
ADD_AND_NORM,
FUSE_ALLREDUCE_MAX_BATCH_SIZE,
)
from sglang.srt.layers.communicator.residual.mhc import (
MHCState,
)
__all__ = [
"ADD_AND_NORM",
"AttentionInputs",
"BoundarySteps",
"CommunicateContext",
"CommunicateSimpleFn",
"CommunicateSummableTensorPairFn",
"DecoderLayerSides",
"EdgeDecl",
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"FfnCompletion",
"FfnExit",
"FfnExitFusion",
"FusedMlpInput",
"HandoffOutput",
"InputRead",
"LayerCommunicator",
"LayerScatterModes",
"LayerStage",
"Layout",
"MHCLayerCommunicator",
"MHCState",
"MixerExit",
"ScatterMode",
"StageDecl",
"StageInput",
"StageOutput",
"SumGroup",
"TokenAxis",
"UnreducedOutput",
"decoder_layer_edges",
"decoder_layer_sides",
"enable_moe_dense_fully_dp",
"get_attn_tp_context",
"input_scattered_layer_sides",
"layer_input_buffer",
"make_boundary",
"moe_cp_gathers_sparse_moe_input",
"move_rows",
"reduce_output",
"sequence_parallel_layer_sides",
"sparse_moe_gathers_over_moe_cp",
"stage_edges",
"tbo_split_moves",
"token_axis_sizes",
"tp_reduce_scatter",
]
@@ -0,0 +1,14 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""What the communicator takes from the modules it serves."""
@@ -0,0 +1,188 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The attention's inputs: the attention-TP context, its cached inputs, and
the attention-TP gather and slice."""
import logging
from contextlib import contextmanager
from typing import Callable
import torch
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
get_local_dp_buffer,
is_dp_attention_enabled,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_forward, get_parallel, get_spec
from sglang.srt.utils import is_cuda, is_npu
_is_cuda = is_cuda()
_is_npu = is_npu()
class AttentionInputs:
def __init__(
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Callable,
*,
is_pre_gathered: bool = False,
):
self.hidden_states_local = hidden_states
self.forward_batch = forward_batch
self.qkv_latent_func = qkv_latent_func
self.hidden_states_ = None
self.qkv_latent_ = None
# When True, hidden_states_local is already attn_tp-gathered upstream
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
self.is_pre_gathered = is_pre_gathered
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
def fetch_qkv_latent(self):
if self.qkv_latent_ is not None:
return self.qkv_latent_
assert self.qkv_latent_func is not None
self.qkv_latent_ = self.qkv_latent_func(
self.hidden_states_local, self.forward_batch
)
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
self.qkv_latent_ = self.tp_all_gather_hidden_states(
self.qkv_latent_, self.forward_batch
)
return self.qkv_latent_
def fetch_hidden_states(self):
if self.hidden_states_ is not None:
return self.hidden_states_
self.hidden_states_ = self.hidden_states_local
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
self.hidden_states_ = self.tp_all_gather_hidden_states(
self.hidden_states_, self.forward_batch
)
return self.hidden_states_
class AttnTpContext:
def __init__(self):
self.allow_input_scattered = False
self.is_dsa = False
def init_context(self, q_lora_rank, is_dsa, is_mhc=False):
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
# cannot use scattered inputs.
self.is_dsa = is_dsa
self.allow_input_scattered = (
get_parallel().enable_attn_tp_input_scattered
and (_is_cuda or _is_npu)
and q_lora_rank is not None
and (is_mhc or not is_dsa)
and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and not enable_moe_dense_fully_dp()
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
and get_spec().speculative_algorithm != "EAGLE3"
)
if get_parallel().enable_attn_tp_input_scattered:
if not self.allow_input_scattered:
logging.info(
"attn_tp_input_scattered is not enabled while other conditions are not met"
)
else:
logging.info("attn_tp_input_scattered is enabled")
def use_input_scattered(self, forward_batch: ForwardBatch):
return (
self.allow_input_scattered
and forward_batch.forward_mode.is_extend()
and not forward_batch.forward_mode.is_target_verify()
and forward_batch.input_ids is not None
and not forward_batch.can_run_tbo
)
@property
def input_scattered(self):
return get_forward().attn_input_scattered
def set_attn_inputs(self, attn_inputs: AttentionInputs):
get_forward().set("attn_inputs", attn_inputs)
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
attn_inputs = get_forward().attn_inputs
if attn_inputs is not None:
attn_inputs.hidden_states_local = hidden_states
def fetch_qkv_latent(self):
attn_inputs = get_forward().attn_inputs
assert attn_inputs is not None
return attn_inputs.fetch_qkv_latent()
def fetch_hidden_states(self):
attn_inputs = get_forward().attn_inputs
assert attn_inputs is not None
return attn_inputs.fetch_hidden_states()
def clear_attn_inputs(self) -> None:
get_forward().set("attn_inputs", None)
@contextmanager
def maybe_input_scattered(self, forward_batch: ForwardBatch):
flag = self.use_input_scattered(forward_batch)
forward = get_forward()
# scoped() also restores when the forward raises — the old in-place
# swap leaked the flag on exceptions.
with forward.scoped(attn_input_scattered=flag):
try:
yield
finally:
forward.set("attn_inputs", None)
ATTN_TP_CONTEXT = AttnTpContext()
def get_attn_tp_context():
return ATTN_TP_CONTEXT
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
gathered = get_local_dp_buffer(
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
)
attn_tp_all_gather_into_tensor(gathered, tensor)
return gathered
def _redistribute_to_attn_tp_shards(
tensor: torch.Tensor, context: CommunicateContext
) -> torch.Tensor:
return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
@@ -0,0 +1,969 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""What each stage produces and needs, and the boundary steps chosen from those
declarations."""
from enum import Enum, auto
from functools import partial
from typing import Callable, FrozenSet, Mapping, Optional, Tuple, Union
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
SumGroup,
TokenAxis,
_gathers_over_attention_cp,
_same_ranks,
_sum_group,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
_hand_qkv_hook_its_input,
_mlp_input_completing_owed,
_mlp_input_dp_partial,
_mlp_input_dp_replicate,
_mlp_input_gather_attention_cp,
_mlp_input_gather_moe_cp,
_mlp_input_norm,
_mlp_input_on_residual_shard,
_mlp_input_residual_into_sum,
_mlp_input_scatter,
_mlp_input_slice,
_mlp_input_without_dp,
tp_reduce_scatter,
)
from sglang.srt.layers.communicator.output import (
HandoffOutput,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.communicator.residual import ResidualOps
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
class StageInput(msgspec.Struct, frozen=True):
"""The rows a stage's consumer needs: sharded over the token axes its
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
class StageOutput(msgspec.Struct, frozen=True):
"""What a stage's producer hands the boundary after it, fixed at
construction: its rows, the group its output is summed over, and when it
leaves that sum to the boundary instead of completing it.
What one output actually owes is carried with the value: a plain
tensor handed across layers is complete, an ``UnreducedOutput`` names what
is left. A raw compute output entering a boundary is read with this
declaration and the boundary's decision for the batch."""
layout: Layout
# None when there is nothing to sum.
group: Optional[SumGroup] = None
# The producer never reduces its output, e.g. a row-parallel projection
# built with reduce_results=False.
always_leaves: bool = False
# Otherwise it leaves the sum only when the boundary publishes the flag for
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
leaves_for_next_layer: bool = False
leaves_for_reduce_scatter: bool = False
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
# combine applies, as a MoE block does without any flag.
leaves_for_reduce_scatterv: bool = False
class DecoderLayerSides(msgspec.Struct, frozen=True):
"""What a decoder layer declares about its attention, its FFN and the rows
between them; ``decoder_layer_edges`` turns it into the layer's
boundaries."""
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageInput
attention_output: StageOutput
ffn: StageInput
ffn_output: StageOutput
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
# The group the layer's input arrives as a partial sum over, if any.
input_owes: Optional[SumGroup] = None
# Whether the residual is added into one rank's share of the attention
# output's sum before that sum completes, instead of after it.
residual_joins_attention_sum: bool = False
class StageDecl(msgspec.Struct, frozen=True):
"""A computing stage's two sides: the rows its input must be on, and what
its output is."""
input: StageInput
output: StageOutput
class EdgeDecl(msgspec.Struct, frozen=True):
"""One boundary between two stages, as the layer that runs one side of it
sees it: what arrives from the producer, what the consumer needs, and the
rows the residual is on before and after the boundary."""
produced: StageOutput
need: StageInput
residual: Layout
residual_to: Layout
# Whether the residual is added into one rank's share of the produced sum
# before that sum completes, instead of after it.
residual_joins_sum: bool = False
class DecoderLayerEdges(msgspec.Struct, frozen=True):
"""A decoder layer's three boundaries: from the rows the previous layer
handed on into the attention, from the attention into the FFN, and from
the FFN onto the rows the layer hands on. The next layer runs the rest of
the last one."""
into_attention: EdgeDecl
into_ffn: EdgeDecl
out_of_ffn: EdgeDecl
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
"""The boundaries of a decoder layer, each with the declarations of its two
sides. What arrives at the layer is complete unless it owes a sum by
construction (``input_owes``); a sum the previous layer leaves for a batch
comes with the value."""
owes = sides.input_owes is not None
arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes)
# Completing what the input owes leaves it and the residual on each
# attention-TP rank's slice.
attention_rows = (
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
if owes
else sides.input_rows
)
return DecoderLayerEdges(
into_attention=EdgeDecl(
produced=arrived,
need=sides.attention,
residual=sides.input_rows,
residual_to=attention_rows,
),
into_ffn=EdgeDecl(
produced=sides.attention_output,
need=sides.ffn,
residual=attention_rows,
residual_to=sides.ffn_residual_rows,
residual_joins_sum=sides.residual_joins_attention_sum,
),
out_of_ffn=EdgeDecl(
produced=sides.ffn_output,
need=StageInput(sides.output_rows),
residual=sides.ffn_residual_rows,
residual_to=sides.output_rows,
),
)
def stage_edges(
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
) -> Tuple[EdgeDecl, EdgeDecl]:
"""The two boundaries of a layer that is one stage of a sequence of stages:
into it from the previous stage's output (None at the start of the layer
stack) and out of it onto ``rows``, the rows every layer hands on and the
residual is on between stages. The previous output arrives on those rows: a
stage whose output is elsewhere (an FFN on the TP group) moves it back there
first, and what it may leave of its sum comes with the value; otherwise the
value is complete. The residual follows the input onto a finer slice and
stays where it is when the input is gathered."""
arrived = (
StageOutput(
rows,
group=previous.group,
always_leaves=previous.always_leaves,
leaves_for_next_layer=previous.leaves_for_next_layer,
)
if previous is not None
and (previous.always_leaves or previous.leaves_for_next_layer)
else StageOutput(rows)
)
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
return (
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
EdgeDecl(
produced=stage.output,
need=StageInput(rows),
residual=during,
residual_to=rows,
),
)
def sequence_parallel_layer_sides(
*, axis_sizes: Mapping[TokenAxis, int]
) -> DecoderLayerSides:
"""A decoder layer while a LayerNorm SP region is active. Its linears
all-gather their input and reduce-scatter their output themselves, so the
layer's rows, its residual and both outputs stay on each TP rank's slice."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather the slices into their GEMM.
attention=StageInput(attention, gathers_itself=gathers),
# o_proj and down reduce-scatter out of theirs, whether fused or not.
attention_output=StageOutput(local),
ffn=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
),
ffn_output=StageOutput(local),
ffn_residual_rows=local,
output_rows=local,
)
def scattered_residual_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
is_first_layer: bool,
is_last_layer: bool,
leaves_for_reduce_scatter: bool,
) -> DecoderLayerSides:
"""A decoder layer on an input-scattered batch whose residual stays on
each attention-TP rank's slice (MHC). The attention and the FFN compute on
the full rows. The attention takes the slice and gathers it itself (its
QKV hook, or the handoff before an attention that needs the rows); the
boundary brings the FFN its rows and moves both outputs back onto the
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
is the embedding's partial sum on the full rows; the last layer hands on
the full rows again."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
return DecoderLayerSides(
input_rows=attention if is_first_layer else local,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
ffn_output=StageOutput(
attention,
group=ffn_group,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
),
ffn_residual_rows=local,
output_rows=attention if is_last_layer else local,
input_owes=SumGroup.TP if is_first_layer else None,
)
def input_scattered_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
hands_on_partial: bool,
) -> DecoderLayerSides:
"""A decoder layer on a batch whose attention input is scattered over
attention TP. Each layer's input is a partial sum over TP on the full rows:
the embedding and every FFN before it leave that sum, and a reduce-scatter
completes it onto each rank's slice, which the attention gathers itself.
The residual comes back to the full rows inside the attention output's sum."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
return DecoderLayerSides(
input_rows=attention,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next layer's input completes the sum; the last layer's FFN does.
ffn_output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
),
ffn_residual_rows=attention,
output_rows=attention,
input_owes=SumGroup.TP,
residual_joins_attention_sum=True,
)
def decoder_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_on_local_rows: bool,
previous_on_local_rows: bool,
is_last_layer: bool,
attention_gathers_local_rows: bool,
ffn_group: SumGroup,
leaves_for_next_layer: bool,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
hands_on_attention_rows: bool = False,
ffn_shards_over_cp: bool = False,
) -> DecoderLayerSides:
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine, or a dense MLP on every rank).
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
computes each CP shard on its own ranks: its rows stay sharded over CP.
A layer on local rows hands on the attention's rows when it is the last, or
with ``hands_on_attention_rows``."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
# Each attention-TP rank's own slice of those rows.
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = (
local
if ffn_on_local_rows
else Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
axis_sizes=axis_sizes,
)
)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
input_rows=local if previous_on_local_rows else attention,
attention=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
attention,
group=SumGroup.ATTN_TP if attention_tp else None,
always_leaves=attention_tp,
),
ffn=StageInput(ffn),
ffn_output=(
StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
group=ffn_group,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
)
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=(
local
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
else attention
),
)
class FusedMlpInput(msgspec.Struct, frozen=True):
"""A kernel that completes the sum the attention output owes together with
the residual add and the post-attention norm, in that order.
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
the residual, or None when it does not take the batch; it returns None only
before touching its inputs or starting a collective."""
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
# It may hand back a new residual and leave the one it took unchanged.
may_return_new_residual: bool
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
"""The moves around the two-batch-overlap split, which cuts the attention's
rows: from the rows the first overlapped layer takes to the attention's,
and back again for each half."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
)
pair = CommunicateSummableTensorPairFn
if layer_input_rows == attention:
return pair._trivial, pair._trivial
if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice: write the residual in and gather over attention
# TP, then take the slice of each half.
return pair._gather, pair._scatter
raise NotImplementedError(f"{layer_input_rows=}")
class CpMoves(msgspec.Struct, frozen=True):
"""How a CP extend's rows reach an FFN that needs all of them and come back,
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
after each rank has completed its own block, ``take_back`` returns this
rank's block of a complete output, and ``reduce_scatter``, where there is
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
returns the block in the same collective."""
gather: Callable
take_back: Callable
reduce_scatter: Optional[Callable] = None
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
def _cp_moves() -> CpMoves:
"""DSA and MLA CP gather equal shards over the attention-CP group and can
complete a sum over it. GQA prefill CP gathers blocks padded to the longest
over the MoE-CP group and takes back only a complete output."""
if _gathers_over_attention_cp():
return CpMoves(
gather=_mlp_input_gather_attention_cp,
take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard,
reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp,
reduce_scatter_group=lambda: get_parallel().attn_cp_group,
)
return CpMoves(
gather=_mlp_input_gather_moe_cp,
take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
)
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps a batch runs at a layer's boundaries: into the attention,
from the attention output to the FFN input, and the FFN output on to the
rows the layer hands on."""
# The half into the attention: completes what the input owes, writes the
# previous output into the residual and reads the attention input
# (_attention_input_step); attention_input then moves it.
attention_prepare: Callable
attention_input: Callable
ffn_input: Callable
# The rows ffn_input hands the FFN.
ffn_input_rows: Layout
# What the FFN exit reads: the FFN output's group and what it may leave.
ffn_output: StageOutput
# The postprocess that moves the FFN output on; None when it goes back over
# attention DP, whose step the FFN exit and postprocess choose per batch.
ffn_output_move: Optional[Callable]
# Whether the next layer's input can take the FFN's sum.
ffn_sum_is_movable: bool
# Whether ffn_output_move also completes the sum the FFN leaves.
ffn_output_move_completes_sum: bool = False
# The fused kernels ffn_input tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Hands the attention its input once attention_input has moved it:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
attention_handoff: Callable = _hand_qkv_hook_its_input
@property
def returns_over_dp(self) -> bool:
return self.ffn_output_move is None
def _attention_input_step(
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
context: "CommunicateContext",
*,
quant_format: str,
post_residual_addition: Optional[torch.Tensor],
layer_input: Optional[Callable],
fusions: Tuple[Callable, ...],
enters_stack: bool,
residual_ops: ResidualOps,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""A boundary's half into an attention: complete what the previous layer
left (a value that owes a sum, or a producer's handoff one of ``fusions``
consumes with the add and norm), what the input owes by construction
(``layer_input``), then write the previous output into the residual and read
the attention input with ``norm``. The layer stack's first layer
(``enters_stack``) starts its residual from its input."""
enters = residual is None and enters_stack
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
if owed is not None and residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
if isinstance(owed, UnreducedOutput) and owed.reduce_and_redistribute is not None:
# No fused kernel runs under attention DP: the reduce-scatter back to
# this rank's tokens comes first.
hidden_states, owed = reduce_output(owed), None
if owed is not None:
for fused in fusions:
result = fused(owed, residual, forward_batch, post_residual_addition)
if result is not None:
return result
if isinstance(owed, HandoffOutput):
# No fused kernel took the handoff: its producer completes it.
hidden_states, owed = reduce_output(owed), None
else:
hidden_states = owed.partial
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual, context)
if enters:
hidden_states, residual = residual_ops.enter(hidden_states), None
if owed is not None and hidden_states.shape[0] != 0:
hidden_states = reduce_output(owed)
if residual is None:
# The previous layer already wrote its output into the residual.
return residual_ops.read_attention_input(hidden_states, norm, quant_format)
return residual_ops.update_and_read_attention_input(
hidden_states, residual, norm, quant_format, post_residual_addition
)
def _another_stage(*args, **kwargs):
"""The steps of a stage a single-stage layer does not have."""
raise RuntimeError("this layer is one stage and does not have the other")
class InputRead(Enum):
"""How a boundary's consumer reads its input from the residual: with the
attention input norm (prepare_attn) or with the FFN input norm and its
fused kernels (prepare_mlp)."""
ATTENTION = auto()
FFN = auto()
class LayerStage(msgspec.Struct, frozen=True):
"""A layer that is one stage of a sequence of stages, each an
attention-like mixer or an FFN: how it reads its input, its two boundaries
(into it, and out of it onto the rows every layer hands on, as
``stage_edges`` gives them), and whether the layer stack
starts at it."""
reads: InputRead
edges: Tuple[EdgeDecl, EdgeDecl]
enters_stack: bool = False
class Boundary(msgspec.Struct, frozen=True):
"""The steps one layer runs at one boundary, chosen from both sides'
declarations. A layer runs the consumer's half of a boundary into one of
its stages, and the producer's half of the boundary after its last stage;
the neighbouring layer runs the other half of that one."""
edge: EdgeDecl
# The consumer's half: completing what the input owes, the add and the
# norm; into the FFN also the move onto the rows it needs.
prepare: Optional[Callable] = None
# Into an attention, the move onto its rows after prepare.
input_move: Optional[Callable] = None
# The fused kernels an FFN's prepare tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# The producer's half: the postprocess that moves the output onto the rows
# the layer hands on; None when it goes back over attention DP, whose step
# the FFN exit and postprocess choose per batch.
output_move: Optional[Callable] = None
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
@property
def input_rows(self) -> Layout:
"""The rows the consumer is handed: what it needs, still sharded over
the axes it gathers itself."""
need = self.edge.need
return Layout(
need.layout.sharded
| (self.edge.produced.layout.sharded & need.gathers_itself)
)
def make_boundary(
edge: EdgeDecl,
*,
reads: Optional[InputRead],
fusions: Tuple = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
enters_stack: bool = False,
) -> Boundary:
"""The steps a layer runs at ``edge``, around the residual operations;
``cp_moves`` for an edge that gathers over or returns across attention CP.
``reads`` is how the consumer reads its input, or None when the consumer
runs in the next layer and ``edge.need`` is the rows this layer hands on:
then only the producer's half runs here. ``fusions`` are the fused kernels
the consumer tries first (FusedMlpInput into an FFN, the attention input's
entries into an attention); ``enters_stack`` for the edge into the layer
stack's first attention. The consumer's half reads only this edge's
declarations, never what the producer chose for a batch; a sum left for a
batch arrives with the value."""
if reads is None:
if edge.need.layout != edge.residual_to:
raise NotImplementedError(f"{edge=}")
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
edge.produced,
residual=edge.residual,
to=edge.residual_to,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
return Boundary(
edge,
output_move=None if returns_over_dp else output_move,
output_move_completes_sum=completes_sum,
)
if reads is InputRead.FFN:
input_step, fused = _select_ffn_input(
edge.produced,
residual=edge.residual,
residual_to=edge.residual_to,
need=edge.need,
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=edge.residual_joins_sum,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
return Boundary(edge, prepare=input_step, fused=fused)
layer_input = None
if edge.produced.always_leaves:
# A reduce-scatter completes the TP sum onto each rank's slice, which the
# attention takes: its group is the TP group without attention DP or CP.
if (
edge.produced.group is not SumGroup.TP
or edge.residual.sharded
or TokenAxis.ATTN_TP_SCATTER not in edge.need.gathers_itself
or edge.residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER}
):
raise NotImplementedError(f"{edge=}")
layer_input = tp_reduce_scatter
return Boundary(
edge,
prepare=partial(
_attention_input_step,
layer_input=layer_input,
fusions=fusions,
enters_stack=enters_stack,
residual_ops=residual_ops,
),
input_move=_select_attention_input_move(edge.residual_to, edge.need),
)
def _select_boundary_steps(
sides: DecoderLayerSides,
*,
fusions: Tuple["FusedMlpInput", ...] = (),
force_layernorm_before_gather: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
attention_handoff: Callable = _hand_qkv_hook_its_input,
attention_fusions: Tuple[Callable, ...] = (),
enters_stack: bool = False,
) -> BoundarySteps:
"""The steps of a decoder layer: its three boundaries, each chosen from the
declarations of its two sides, around the layer's residual operations. Both
edges that cross attention CP take the same ``cp_moves``; the attention's
input tries ``attention_fusions``, and the layer stack's first layer
(``enters_stack``) starts its residual there."""
edges = decoder_layer_edges(sides)
out_of_ffn = make_boundary(
edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops
)
into_ffn = make_boundary(
edges.into_ffn,
reads=InputRead.FFN,
fusions=fusions,
force_layernorm_before_gather=force_layernorm_before_gather,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
into_attention = make_boundary(
edges.into_attention,
reads=InputRead.ATTENTION,
fusions=attention_fusions,
residual_ops=residual_ops,
enters_stack=enters_stack,
)
return BoundarySteps(
attention_prepare=into_attention.prepare,
attention_input=into_attention.input_move,
ffn_input=into_ffn.prepare,
ffn_input_rows=into_ffn.input_rows,
ffn_output=edges.out_of_ffn.produced,
ffn_output_move=out_of_ffn.output_move,
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
fused=into_ffn.fused,
attention_handoff=attention_handoff,
)
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
"""How the rows a layer takes become its attention's input: as they are,
or gathered over attention TP from each rank's slice, unless the attention
gathers them itself."""
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
if not need.layout.sharded <= rows.sharded or gathered not in (
frozenset(),
{TokenAxis.ATTN_TP_SCATTER},
):
raise NotImplementedError(f"{rows=} {need=}")
if gathered:
return CommunicateSimpleFn._scattered_to_tp_attn_full
return CommunicateSimpleFn._trivial
def _select_ffn_input(
produced: StageOutput,
*,
residual: Layout,
residual_to: Layout,
need: StageInput,
force_layernorm_before_gather: bool,
fusions: Tuple[FusedMlpInput, ...],
residual_joins_sum: bool = False,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
"""The steps from the attention output to the FFN input, and the fused
kernels they try first: complete the attention-TP sum, move the residual to
the rows it has while the FFN runs, write the output into it and read the
FFN input, and bring the rows to what the FFN's group needs: a gather over
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
only when nothing is gathered or sliced, and only if it completes the sum
the attention output owes. A write-back that is not a plain add runs only
after the sum completes."""
if produced.leaves_for_next_layer and produced.group not in (
None,
SumGroup.ATTN_TP,
):
# A producer that leaves its sum only for some batches (an FFN before
# this one) hands that sum on with the value: complete it, then the
# input is a complete output.
step, fused = _select_ffn_input(
StageOutput(produced.layout),
residual=residual,
residual_to=residual_to,
need=need,
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=residual_joins_sum,
cp_moves=cp_moves,
residual_ops=residual_ops,
)
return partial(_mlp_input_completing_owed, step=step), fused
# What the attention output owes decides the steps: the attention-TP sum,
# always left by the output projection, or nothing.
owes_attention_tp = produced.group is SumGroup.ATTN_TP
if owes_attention_tp != produced.always_leaves or produced.group not in (
None,
SumGroup.ATTN_TP,
):
raise NotImplementedError(f"{produced=}")
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
sliced = need.layout.sharded - produced.layout.sharded
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective; a
# complete value is only sliced.
if (
sliced != {TokenAxis.ATTN_TP_SCATTER}
or gathered
or residual_to != need.layout
or residual not in (produced.layout, need.layout)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return (
partial(
_mlp_input_scatter if owes_attention_tp else _mlp_input_slice,
scatters_residual=residual != residual_to,
residual_ops=residual_ops,
),
(),
)
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
# The residual stays on each rank's slice while the FFN takes the
# attention's rows (MHC on an input-scattered batch).
if gathered or not owes_attention_tp or residual != residual_to:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
if gathered == {TokenAxis.ATTN_CP}:
# Each CP rank completes its own chunk, then the CP moves gather them.
if cp_moves is None:
raise NotImplementedError(f"{produced=} {need=}")
on_chunk, fused = _select_ffn_input(
produced,
residual=residual,
residual_to=residual_to,
need=StageInput(produced.layout),
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=residual_joins_sum,
residual_ops=residual_ops,
)
return partial(cp_moves.gather, gather=on_chunk), fused
if (
residual_to != produced.layout
or gathered
not in (
frozenset(),
{TokenAxis.ATTN_DP},
{TokenAxis.ATTN_DP, TokenAxis.ATTN_CP},
)
or residual.sharded - residual_to.sharded
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
# A residual arriving on each rank's slice is gathered back first.
gathers_residual = residual != residual_to
if not gathered:
if not owes_attention_tp:
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
if not residual_ops.adds_plainly:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return _mlp_input_residual_into_sum, ()
fused = tuple(f for f in fusions if f.completes is produced.group)
return (
partial(
_mlp_input_without_dp,
gathers_residual=gathers_residual,
fusions=tuple(f.run for f in fused),
residual_ops=residual_ops,
),
fused,
)
# The partial order adds the residual on attention-TP rank 0 before the DP
# gather's collective completes that sum, which only a plain residual add
# allows.
# Over attention DP and CP, the DP gather puts each CP rank's shard in its
# DP group's slot, so the one DP sum gathers both axes.
places_cp_shards = TokenAxis.ATTN_CP in gathered
if (
owes_attention_tp
and not force_layernorm_before_gather
and residual_ops.adds_plainly
):
return (
partial(
_mlp_input_dp_partial,
gathers_residual=gathers_residual,
places_cp_shards=places_cp_shards,
),
(),
)
return (
partial(
_mlp_input_dp_replicate,
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
places_cp_shards=places_cp_shards,
residual_ops=residual_ops,
),
(),
)
def _select_ffn_output_move(
produced: StageOutput,
*,
residual: Layout,
to: Layout,
cp_moves: Optional[CpMoves] = None,
residual_ops: ResidualOps = ADD_AND_NORM,
) -> Tuple[bool, Optional[Callable], bool]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
that step), or else the postprocess that moves it, None when there is none
to choose here; and whether that move also completes the sum the FFN
leaves."""
pair = CommunicateSummableTensorPairFn
if produced.layout == residual:
if to == residual:
return False, pair._trivial, False
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: write the output
# into the residual, then gather over attention TP.
return False, partial(pair._gather, residual_ops=residual_ops), False
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
# The residual stays on each rank's slice (MHC on an input-scattered
# batch): a reduce-scatter onto the slice completes the sum the FFN
# leaves; a complete output is only sliced.
sums = produced.leaves_for_reduce_scatter
return (
False,
partial(
pair._onto_residual_shard,
sums=sums,
gathers_back=to != residual,
residual_ops=residual_ops,
),
sums,
)
if to != residual or not produced.layout.sharded <= residual.sharded:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
if returned == {TokenAxis.ATTN_CP}:
if cp_moves is None:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
if not produced.leaves_for_reduce_scatter:
# A complete output: this rank's block of it, nothing summed.
return False, cp_moves.take_back, False
# The FFN leaves its sum: only a take-back that sums over the same
# ranks completes it.
if cp_moves.reduce_scatter is None or not _same_ranks(
_sum_group(produced.group), cp_moves.reduce_scatter_group()
):
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return False, cp_moves.reduce_scatter, True
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
# This rank's CP shard, from where the DP gather put it.
return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False
if returned != {TokenAxis.ATTN_DP}:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return True, None, False
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,251 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Token axes, layouts and process groups of what a layer hands across its
boundaries, and the configuration and batch facts they depend on."""
from dataclasses import dataclass
from enum import Enum, auto
from typing import Dict, FrozenSet, List, Mapping, Optional
import msgspec
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.attention.dsa.utils import (
dsa_use_prefill_cp,
is_dsa_enable_prefill_cp,
)
from sglang.srt.layers.cp.utils import is_mla_cp_active, is_mla_cp_enabled
from sglang.srt.layers.dp_attention import (
get_moe_cp_size,
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.moe import (
is_moe_input_scattered_across_dp_ranks,
post_experts_reduction_group,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
class TokenAxis(Enum):
"""A parallel dimension across which ranks hold different tokens."""
ATTN_DP = auto()
ATTN_CP = auto()
# Each attention-TP rank holds a slice of its group's tokens.
ATTN_TP_SCATTER = auto()
class Layout(msgspec.Struct, frozen=True):
"""The token axes a rank's rows are sharded over. Single-rank axes are left
out, so two layouts are equal exactly when every rank holds the same tokens."""
sharded: FrozenSet[TokenAxis]
@classmethod
def sharded_over(
cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int]
) -> "Layout":
return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1))
class SumGroup(Enum):
"""The group a stage's output is summed over, by name: the handle comes
from get_parallel() when the sum runs."""
ATTN_TP = auto()
TP = auto()
# The group one all-reduce of a MoE output runs over.
MOE_OUTPUT = auto()
class ScatterMode(Enum):
"""
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
Model input/output: [ab, ab, cd, cd] for four ranks respectively
SCATTERED: [a, b, c, d]
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
FULL: [abcd, abcd, abcd, abcd]
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
"""
SCATTERED = auto()
TP_ATTN_FULL = auto()
FULL = auto()
MOE_FULL = auto()
@staticmethod
def model_input_output():
"""The scatter mode for model forward pass input and output data"""
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
return ScatterMode.SCATTERED
return ScatterMode.TP_ATTN_FULL
def enable_moe_dense_fully_dp():
return get_parallel().moe_dense_tp_size == 1
def _generic_prefill_cp_shards_tokens() -> bool:
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
parallel = get_parallel()
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
def enable_dwdp():
return get_parallel().dwdp_size > 1
def _batch_size(forward_batch: ForwardBatch) -> int:
return (
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
)
def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
if is_dp_attention_enabled():
# The FFN runs on every DP rank's tokens, so every rank decides alike.
return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0
return _batch_size(forward_batch) > 0
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
attn_tp_rank: int
attn_tp_size: int
attn_dp_size: int
attn_cp_rank: int
attn_cp_size: int
tp_size: int
cache = None
tp_rank: int
force_layernorm_before_dp_gather: bool = False
@classmethod
def init_new(cls):
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
attn_dp_size = get_parallel().attn_dp_size
attn_cp_size = get_parallel().attn_cp_size
attn_cp_rank = get_parallel().attn_cp_rank
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
moe_cp_size = get_moe_cp_size()
process_group_sizes = {
ScatterMode.SCATTERED: 1,
ScatterMode.TP_ATTN_FULL: attn_tp_size,
# TODO: support --moe-dense-tp-size > 1
# With context parallel enabled, we should exclude
# the attn_cp_size from the total tp_size
ScatterMode.FULL: tp_size // attn_cp_size,
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
}
return cls(
process_group_sizes=process_group_sizes,
attn_tp_rank=attn_tp_rank,
attn_tp_size=attn_tp_size,
attn_dp_size=attn_dp_size,
attn_cp_rank=attn_cp_rank,
attn_cp_size=attn_cp_size,
tp_size=tp_size,
tp_rank=tp_rank,
)
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
gathered across the MoE-CP group, or None when it is not: only a context
parallel extend with CP metadata gathers, and only when the group has more
than one rank. The batch is read first, so a batch that is not a CP extend
never reads the group."""
if (
forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
and get_moe_cp_size() > 1
):
return forward_batch.attn_cp_metadata.per_rank_actual_token
return None
def sparse_moe_gathers_over_moe_cp() -> bool:
"""Whether a sparse MoE's input is gathered over the MoE-CP group on a CP
extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than
the MoE's data-parallel groups. DSA and MLA CP gather over attention CP
instead."""
return (
not is_moe_input_scattered_across_dp_ranks()
and is_enable_moe_cp_allgather()
and not _gathers_over_attention_cp()
)
def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool:
"""Whether a sparse MoE's input is gathered over the MoE-CP group on this
batch: a CP extend, for a MoE that gathers there."""
return (
sparse_moe_gathers_over_moe_cp()
and moe_cp_gathered_rows(forward_batch) is not None
)
def _cp_shard_token_rows(forward_batch: ForwardBatch) -> List[int]:
"""Rows of each CP rank's shard that hold tokens; the shards are padded to
one length after them."""
metadata = forward_batch.attn_cp_metadata
return metadata.per_rank_logical_token or metadata.per_rank_actual_token
def _sum_group(group: SumGroup) -> GroupCoordinator:
parallel = get_parallel()
if group is SumGroup.ATTN_TP:
return parallel.attn_tp_group
if group is SumGroup.TP:
return parallel.tp_group
return post_experts_reduction_group()
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
"""The token axes' sizes for a batch: attention CP shards tokens only on
a CP extend (``cp_active``); otherwise every CP rank holds them all."""
parallel = get_parallel()
return {
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
}
def _gathers_over_attention_cp() -> bool:
"""Whether a CP extend gathers the FFN input over the attention-CP group in
equal shards and takes the output back with a reduce-scatter there: DSA and
MLA CP. GQA prefill CP gathers over the MoE-CP group instead."""
return is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
def _batch_shards_over_cp(forward_batch: ForwardBatch) -> bool:
"""Whether this batch's tokens are split across the CP ranks. Only a context
parallel extend is, so other batches, decode graph capture among them,
never read the CP predicates."""
if not forward_batch.forward_mode.is_context_parallel_extend():
return False
if _gathers_over_attention_cp():
return dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch)
return moe_cp_gathered_rows(forward_batch) is not None
def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool:
return sorted(a.ranks) == sorted(b.ranks)
@@ -0,0 +1,794 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The communication a boundary runs: the attention input, the FFN input and the
FFN output's way back, over attention TP, DP and CP."""
from typing import Callable, List, Optional, Tuple, Union
import torch
from sglang.srt.distributed import (
GroupCoordinator,
attention_tensor_model_parallel_all_reduce,
attention_tensor_model_parallel_quant_all_reduce,
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
TokenAxis,
_cp_shard_token_rows,
moe_cp_gathered_rows,
)
from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output
from sglang.srt.layers.communicator.residual import ResidualOps
from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM
from sglang.srt.layers.communicator_dsa_cp import (
dsa_cp_gather_hidden_states,
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
can_use_dp_reduce_scatter,
dp_gather_partial,
dp_gather_replicate,
dp_reduce_scatter_tensor,
dp_scatter,
get_dp_global_num_tokens,
get_global_dp_buffer,
get_local_dp_buffer,
get_moe_cp_rank,
get_moe_cp_size,
is_allocation_symmetric,
moe_cp_all_gather_into_tensor,
)
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
is_npu,
)
_is_npu = is_npu()
if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
def tp_reduce_scatter(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
context: "CommunicateContext",
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Module-level so MHC communicators can reuse it without a
``LayerCommunicator`` instance."""
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
assert hidden_states.shape[0] % context.tp_size == 0, (
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
)
local_tokens = hidden_states.shape[0] // context.tp_size
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
if residual is not None:
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
return output, residual
def layer_input_buffer(
hidden_states: Union[torch.Tensor, UnreducedOutput],
) -> torch.Tensor:
"""The tensor holding a layer's input, for reusing its memory without reading it."""
if isinstance(hidden_states, UnreducedOutput):
return hidden_states.partial
return hidden_states
class CommunicateSimpleFn:
@staticmethod
def _trivial(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
) -> torch.Tensor:
return hidden_states
@staticmethod
def _scattered_to_tp_attn_full(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
context: CommunicateContext,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
for local_hidden_states in hidden_states:
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
output = torch.empty(
(
local_hidden_states.shape[0] * context.attn_tp_size,
*local_hidden_states.shape[1:],
),
dtype=local_hidden_states.dtype,
device=local_hidden_states.device,
)
attn_tp_all_gather_into_tensor(
output,
local_hidden_states,
)
gathered_hidden_states.append(output)
return tuple(gathered_hidden_states)
return _redistribute_from_attn_tp_shards(hidden_states)
def _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states: torch.Tensor, context: CommunicateContext
) -> torch.Tensor:
local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
]
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
return local_hidden_states
def _redistribute_input_to_moe_cp(
hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int
) -> torch.Tensor:
# Zigzag split can produce unequal token counts across CP ranks
# (when seq_len % (cp_size * 2) != 0). NCCL allgather requires
# equal input sizes, so pad to the max per-rank token count.
max_tokens = max(rows)
pad_size = max_tokens - hidden_states.shape[0]
if pad_size > 0:
hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size])
output = torch.empty(
(max_tokens * moe_cp_size, hidden_states.shape[1]),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
moe_cp_all_gather_into_tensor(output, hidden_states)
return output
def _mlp_input_reduce_output(
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
) -> torch.Tensor:
if (
may_quantize
and not forward_batch.forward_mode.is_decode_or_idle()
and get_exec().comm.enable_quant_communications
):
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
return attention_tensor_model_parallel_all_reduce(hidden_states)
def _redistribute_input_to_dp(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
cp_shard_counts: Optional[List[int]] = None,
) -> torch.Tensor:
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
dp_gather_replicate(
global_hidden_states, hidden_states, forward_batch, cp_shard_counts
)
return global_hidden_states
def _reduce_and_redistribute_output_to_dp(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
cp_shard_counts: Optional[List[int]] = None,
) -> torch.Tensor:
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
dp_gather_partial(
global_hidden_states, hidden_states, forward_batch, cp_shard_counts
)
return global_hidden_states
def move_rows(
hidden_states: torch.Tensor,
rows: Layout,
to: Layout,
forward_batch: ForwardBatch,
) -> torch.Tensor:
"""A complete value from the rows it is on to ``to``: gathered over the
token axes ``to`` does not shard (attention TP, then attention DP), or cut to
this rank's share of those it does (attention DP, then attention TP)."""
gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded
if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut:
raise NotImplementedError(f"{rows=} {to=}")
if TokenAxis.ATTN_TP_SCATTER in gathered:
hidden_states = _redistribute_from_attn_tp_shards(hidden_states)
if TokenAxis.ATTN_DP in gathered:
hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch)
if TokenAxis.ATTN_DP in cut:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
if TokenAxis.ATTN_TP_SCATTER in cut:
parallel = get_parallel()
hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
parallel.attn_tp_rank
]
return hidden_states
def _mlp_input_without_dp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
residual_ops: ResidualOps = ADD_AND_NORM,
):
if gathers_residual:
residual = residual_ops.residual_from_attn_tp_shards(residual)
for fused in fusions:
result = fused(hidden_states, residual, forward_batch)
if result is not None:
return result
# MHC sums its streams in full precision.
hidden_states = _mlp_input_reduce_output(
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_dp_replicate(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
residual_ops: ResidualOps = ADD_AND_NORM,
):
"""Attention DP: complete the attention-TP sum if it is owed, write the
output into the residual and read the FFN input locally, then gather. With
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
beside the others' in the group's slot."""
if gathers_residual:
residual = residual_ops.residual_from_attn_tp_shards(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
):
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
)
else:
hidden_states, residual = residual_ops.update_and_read_ffn_input(
hidden_states, residual, layernorm
)
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
return (
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
residual,
)
def _mlp_input_dp_partial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gathers_residual: bool,
places_cp_shards: bool = False,
):
"""Attention DP: one rank adds the residual, the gather sums it, then
normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP
group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if context.attn_tp_rank == 0:
hidden_states += residual
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
hidden_states = _reduce_and_redistribute_output_to_dp(
hidden_states, forward_batch, cp_shard_counts
)
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
if hidden_states.shape[0] != 0:
hidden_states = layernorm(hidden_states)
return hidden_states, residual
def _hand_qkv_hook_its_input(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Give the attention's QKV hook the attention input."""
if qkv_latent_func is not None:
get_attn_tp_context().set_attn_inputs(
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
)
return hidden_states
def _hand_scattered_input_to_attention(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
qkv_latent_func: Optional[Callable],
) -> torch.Tensor:
"""Input-scattered attention takes each rank's slice, and its QKV hook
gathers the rows after the projection. DSA and attention without a hook
consume full hidden states, so those are gathered here, and the hook is
told they are."""
ctx = get_attn_tp_context()
if ctx.is_dsa or qkv_latent_func is None:
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
if qkv_latent_func is not None:
ctx.set_attn_inputs(
AttentionInputs(
hidden_states,
forward_batch,
qkv_latent_func,
is_pre_gathered=ctx.is_dsa,
)
)
return hidden_states
def _tp_all_gather_scattered_rows(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
# Input-scattered attention keeps the same number of tokens on every TP rank.
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
def _mlp_input_completing_owed(
hidden_states: Union[torch.Tensor, "UnreducedOutput"],
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
step: Callable,
):
"""An FFN input whose producer may have left its sum: complete what the
value owes, then run ``step`` on the complete output."""
return step(
reduce_output(hidden_states), residual, forward_batch, layernorm, context
)
def _mlp_input_norm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps = ADD_AND_NORM,
):
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
if scatters_residual:
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_slice(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
scatters_residual: bool,
residual_ops: ResidualOps = ADD_AND_NORM,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
if scatters_residual and residual is not None:
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
def _mlp_input_on_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
residual_ops: ResidualOps,
):
"""The residual stays on each rank's slice while the FFN takes the full
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
attn_tp_all_gather_into_tensor(hidden_states, shard)
return hidden_states, residual
def _mlp_input_residual_into_sum(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context
)
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = layernorm(residual)
return hidden_states, residual
def _mlp_input_gather_attention_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gather: Callable,
):
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context
)
return dsa_cp_gather_hidden_states(hidden_states), residual
def _mlp_input_gather_moe_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
gather: Callable,
):
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
TP_ATTN_FULL."""
# Early return on empty tensor is safe for MOE_CP because:
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
# so no rank hits this path while others proceed to the allgather.
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
if hidden_states.shape[0] == 0:
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context
)
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None and hidden_states.shape[0] > 0:
hidden_states = _redistribute_input_to_moe_cp(
hidden_states, rows, get_moe_cp_size()
)
return hidden_states, residual
def _dp_scatter_group() -> GroupCoordinator:
parallel = get_parallel()
if parallel.tp_size == parallel.attn_dp_size:
return parallel.tp_group
return parallel.attn_tp_group
def _reduce_and_redistribute_output_varlen(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
get_parallel().tp_group.reduce_scatterv(
hidden_states,
output=local_hidden_states,
sizes=get_dp_global_num_tokens(),
)
def _reduce_and_redistribute_output_max_len(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
dp_reduce_scatter_tensor(local_hidden_states, hidden_states)
def _redistribute_output(
local_hidden_states: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> None:
dp_scatter(local_hidden_states, hidden_states, forward_batch)
def _redistribute_output_from_moe_cp(
hidden_states: torch.Tensor, rows: List[int]
) -> torch.Tensor:
moe_cp_rank = get_moe_cp_rank()
# The allgather was padded to max_tokens_per_rank (equal chunks).
# Extract this rank's actual (non-padded) tokens from its chunk.
max_tokens_per_rank = max(rows)
actual_local_tokens = rows[moe_cp_rank]
return hidden_states.narrow(
0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens
).contiguous()
def _reduce_and_redistribute_output_step(
forward_batch: ForwardBatch,
*,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings an FFN output gathered over attention
DP back to this rank's tokens when the FFN leaves its sum to it (see
StageOutput); None when the FFN reduces the output and only a scatter
remains."""
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
return _reduce_and_redistribute_output_varlen
if (
leaves_for_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
return _reduce_and_redistribute_output_max_len
return None
def _to_local_tokens(
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
forward_batch: ForwardBatch,
hidden_states: torch.Tensor,
) -> torch.Tensor:
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
step(local_hidden_states, hidden_states, forward_batch)
return local_hidden_states
def _all_reduce_then_to_local_tokens(
group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor
) -> torch.Tensor:
return _to_local_tokens(
_redistribute_output, forward_batch, group.all_reduce(hidden_states)
)
class CommunicateSummableTensorPairFn:
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
@staticmethod
def _trivial(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
return hidden_states, residual
@staticmethod
def _scatter_hidden_states(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
allow_reduce_scatter: bool = False,
is_layer_sparse: bool = False,
):
step = (
_reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=allow_reduce_scatter,
# A MoE block leaves its sum to reduce_scatterv whenever it
# applies (should_skip_post_experts_all_reduce).
leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
)
or _redistribute_output
)
return _to_local_tokens(step, forward_batch, hidden_states), residual
@staticmethod
def _gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
residual_ops: ResidualOps = ADD_AND_NORM,
**kwargs,
):
hidden_states = residual_ops.update_residual(hidden_states, residual)
return _redistribute_from_attn_tp_shards(hidden_states), None
@staticmethod
def _onto_residual_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
sums: bool,
gathers_back: bool,
residual_ops: ResidualOps,
**kwargs,
):
"""Bring the FFN output onto the slice of the rows the residual is on:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
if sums:
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
else:
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
if not gathers_back:
return hidden_states, residual
local_states = residual_ops.update_residual(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
)
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
def _scatter(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
):
assert residual is None, "not yet handled residual!=None"
return _redistribute_to_attn_tp_shards(hidden_states, context), None
@staticmethod
def _take_back_attention_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
equal shards over the attention-CP group."""
parallel = get_parallel()
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
return shard, residual
@staticmethod
def _reduce_scatter_over_cp(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
keep this rank's shard."""
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
@staticmethod
def _take_back_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
slot, at the shard's padded length with the padding zeroed."""
held = forward_batch.attn_cp_metadata.per_rank_actual_token
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
: held[get_parallel().attn_cp_rank]
]
dp_scatter(
local_hidden_states,
hidden_states,
forward_batch,
_cp_shard_token_rows(forward_batch),
)
return local_hidden_states, residual
@staticmethod
def _scatter_hidden_states_moe(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
CP-local portion.
If DP>1, further scatter back to the local DP slice.
"""
# Only scatter back during prefill; decode was never allgathered so no-op.
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
# — CP extend always has non-zero tokens per rank, and decode skips this path.
rows = moe_cp_gathered_rows(forward_batch)
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
if context.attn_dp_size > 1:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
return hidden_states, residual
@@ -0,0 +1,59 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Outputs that carry work their consumer still owes."""
from typing import Callable, Optional, Union
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
it any other way. The producer says what is owed: one all-reduce over
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
partial: torch.Tensor
group: Optional[GroupCoordinator] = None
# Under attention DP: the reduction that also brings ``partial`` back to this
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
class HandoffOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes work only its producer knows how to do (a
MoE's finalize and sum), left for the next layer's input. A fused kernel
there may do that work together with its own; anything else passes it
through reduce_output(), which calls ``complete()``."""
def complete(self) -> torch.Tensor:
"""Do the owed work, unfused, and return the complete output."""
raise NotImplementedError
def reduce_output(
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None],
) -> Optional[torch.Tensor]:
"""Run the work an UnreducedOutput or a HandoffOutput still owes; pass
anything else through."""
if isinstance(hidden_states, UnreducedOutput):
if hidden_states.reduce_and_redistribute is not None:
return hidden_states.reduce_and_redistribute(hidden_states.partial)
return hidden_states.group.all_reduce(hidden_states.partial)
if isinstance(hidden_states, HandoffOutput):
return hidden_states.complete()
return hidden_states
@@ -0,0 +1,60 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""How a layer writes its stages' outputs into the residual and reads the next
stage's input back."""
from typing import Protocol, Tuple
import torch
class ResidualOps(Protocol):
"""How a layer writes each stage's output into its residual and reads the
next stage's input from it. The boundary steps complete sums and move
tokens around these operations. AddAndNorm adds and normalizes; MHC's
hyper-connection streams implement the same operations (residual.mhc)."""
# The write-back is a plain add, which one rank may run before the sum it
# adds into completes: the DP partial order, and a residual that joins the
# attention output's sum.
adds_plainly: bool
# The layer writes its FFN output into the residual itself instead of
# leaving that to the next layer's input.
updates_residual_after_ffn: bool
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""The residual the layer stack starts from, given its input."""
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
"""The attention input and the residual, from a residual that already
holds the previous layer's output."""
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
) -> Tuple:
"""Write the previous layer's output into the residual, then read the
attention input from it."""
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
"""Write the attention output into the residual, then read the FFN
input from it."""
def update_residual(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the FFN output written into it."""
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
"""This attention-TP rank's slice of the residual."""
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
"""The residual gathered from every attention-TP rank's slice."""
@@ -0,0 +1,311 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The plain residual: add and RMSNorm, and the fused all-reduce + add + norm
kernels."""
from typing import Optional
import torch
from sglang.srt.layers.communicator.adapters.attention import (
_redistribute_from_attn_tp_shards,
_redistribute_to_attn_tp_shards,
get_attn_tp_context,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.flashinfer_comm_fusion import (
is_flashinfer_allreduce_unavailable,
uses_cutedsl_ar_fusion,
)
from sglang.srt.layers.quantization.fp8_utils import (
_use_aiter_bpreshuffle_gfx95,
materialize_bpreshuffle_fp8_scale_tuple,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_exec, get_parallel, get_platform
from sglang.srt.utils import (
get_bool_env_var,
is_cuda,
is_flashinfer_available,
is_gfx95_supported,
is_gfx1250_supported,
is_hip,
)
_is_cuda = is_cuda()
_is_flashinfer_available = is_flashinfer_available()
_is_sm90_supported = _is_cuda and get_platform().is_sm90
_is_sm100_supported = _is_cuda and get_platform().is_sm100
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
_is_gfx95_supported = is_gfx95_supported()
_is_gfx1250_supported = is_gfx1250_supported()
if _use_aiter:
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype
if _is_gfx1250_supported:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
else:
from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant
from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant
if _is_gfx95_supported:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
from sglang.srt.layers.quantization.rocm_mxfp4_utils import (
fused_rms_mxfp4_quant,
)
def _fused_rmsnorm_fp8_per_token_quant(
hidden_states: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
residual: Optional[torch.Tensor] = None,
):
"""Fused (optional residual-add +) RMSNorm + FP8 per-token quantization.
Only used with the aiter (ROCm) backend.
Args:
residual: if provided, computes hidden_states + residual before RMSNorm
and returns updated residual_out as second element.
Returns:
If residual is None: (out_fp8, scale)
If residual provided: ((out_fp8, scale), residual_out)
"""
if _is_gfx1250_supported:
# per-token quant == group quant with group_size == hidden size, giving
# an (M, 1) scale.
N = hidden_states.shape[-1]
(out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant(
hidden_states,
weight,
epsilon,
group_size=N,
dtype_quant=_aiter_fp8_dtype,
res1=residual,
)
if residual is not None:
return (out_fp8, scale), residual_out
return (out_fp8, scale)
M, N = hidden_states.shape
out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device)
scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device)
if residual is not None:
residual_out = torch.empty_like(hidden_states)
_aiter_add_rmsnorm_quant(
out_fp8,
hidden_states,
residual,
residual_out,
scale,
weight,
epsilon,
0, # group_size=0 → per-token
)
return (out_fp8, scale.unsqueeze(1)), residual_out
else:
_aiter_rmsnorm_quant(
out_fp8,
hidden_states,
scale,
weight,
epsilon,
0, # group_size=0 → per-token
)
return (out_fp8, scale.unsqueeze(1))
# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465
# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True).
FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048
def apply_flashinfer_allreduce_fusion(batch_size: int):
return (
# NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion
# Ref: https://github.com/sgl-project/sglang/issues/17237
(_is_sm90_supported or _is_sm100_supported)
and _is_flashinfer_available
and not is_dp_attention_enabled()
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
# cutedsl runs its own fused path from the fusion communicator.
and not uses_cutedsl_ar_fusion()
and not is_flashinfer_allreduce_unavailable()
# Symbolic size checks stay last: under Dynamo tracing they guard on
# the dynamic token dim, so statically-off configs must short-circuit
# before reaching them.
and batch_size > 0
and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE
)
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
comm = get_exec().comm
if not comm.enable_aiter_allreduce_fusion:
return False
if forward_mode.is_extend_or_draft_extend_or_mixed():
return not comm.disable_aiter_allreduce_fusion_in_prefill
return not comm.disable_aiter_allreduce_fusion_in_decode
def apply_aiter_all_reduce_fusion(
input_tensor: torch.Tensor, forward_batch: ForwardBatch
):
n = input_tensor.shape[-1]
total_bytes = input_tensor.numel() * input_tensor.element_size()
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
return (
_use_aiter
and total_bytes > 0
and n <= 16384
and total_bytes <= 8 * 1024 * 8192
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
def _update_and_read_residual_plain(
norm, hidden_states, residual, post_residual_addition
):
if residual is None:
return norm(hidden_states), hidden_states
return norm(hidden_states, residual, post_residual_addition)
def _update_and_read_residual_aiter_mxfp4(
norm, hidden_states, residual, post_residual_addition
):
# post_residual_addition is not applied on this path.
output, *_, residual_out = fused_rms_mxfp4_quant(
hidden_states,
norm.weight,
norm.variance_epsilon,
None,
None,
None,
residual,
)
return output, hidden_states if residual is None else residual_out
def _update_and_read_residual_aiter_fp8_group(
norm, hidden_states, residual, post_residual_addition
):
"""aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the
unquantized bf16 output rides along as a third element, so the DSA indexer
can skip dequantizing. post_residual_addition is not applied on this path."""
needs_bf16 = get_attn_tp_context().is_dsa
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
hidden_states,
norm.weight,
norm.variance_epsilon,
inp2=None,
inp2_weight=None,
inp2_epsilon=None,
group_size=128,
dtype_quant=torch.float8_e4m3fn,
res1=residual,
output_unquantized_inp1=needs_bf16,
transpose_scale=False,
)
if _use_aiter_bpreshuffle_gfx95:
output = materialize_bpreshuffle_fp8_scale_tuple(output)
if needs_bf16:
output = (output[0], output[1], unquantized)
return output, hidden_states if residual is None else residual_out
def _update_and_read_residual_aiter_fp8_per_token(
norm, hidden_states, residual, post_residual_addition
):
if residual is None:
output = _fused_rmsnorm_fp8_per_token_quant(
hidden_states, norm.weight.data, norm.variance_epsilon
)
return output, hidden_states
if post_residual_addition is not None:
residual = residual + post_residual_addition
return _fused_rmsnorm_fp8_per_token_quant(
hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual
)
def _attn_input_update_and_read_residual(quant_format: str):
"""Add the previous layer's output to the residual and read the attention
input from it: the input norm, fused with the quantization this format
wants. Without a residual (the first layer, or one already folded into
hidden_states) the input itself becomes the residual."""
if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format:
return _update_and_read_residual_aiter_mxfp4
if _use_aiter and _is_gfx95_supported and quant_format == "fp8":
return _update_and_read_residual_aiter_fp8_group
if _use_aiter and quant_format == "fp8_per_token":
return _update_and_read_residual_aiter_fp8_per_token
return _update_and_read_residual_plain
class AddAndNorm:
"""A plain residual: a stage's output is added into it, and the next
stage's input is its norm. An empty batch skips the norms."""
adds_plainly = True
updates_residual_after_ffn = False
def enter(self, hidden_states):
return hidden_states
def read_attention_input(self, residual, norm, quant_format):
if residual.shape[0] == 0:
return residual, residual
return _attn_input_update_and_read_residual(quant_format)(
norm, residual, None, None
)
def update_and_read_attention_input(
self, hidden_states, residual, norm, quant_format, post_residual_addition
):
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return _attn_input_update_and_read_residual(quant_format)(
norm, hidden_states, residual, post_residual_addition
)
def update_and_read_ffn_input(self, hidden_states, residual, norm):
if residual is None:
# The layer stack starts at this FFN: its input is the residual.
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
return norm(hidden_states), hidden_states
if hidden_states.shape[0] == 0:
return hidden_states, residual
return norm(hidden_states, residual)
def update_residual(self, hidden_states, residual):
hidden_states += residual
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
return _redistribute_to_attn_tp_shards(residual, context)
def residual_from_attn_tp_shards(self, residual):
return _redistribute_from_attn_tp_shards(residual)
ADD_AND_NORM = AddAndNorm()
@@ -11,19 +11,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Hyper-connection residual streams."""
from dataclasses import dataclass
from typing import Callable, Optional
import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis
from sglang.srt.layers.communicator import (
BoundarySteps,
LayerCommunicator,
LayerScatterModes,
sparse_moe_gathers_over_moe_cp,
)
@dataclass
@@ -141,66 +136,3 @@ class MHCState:
raise NotImplementedError(
"Unsupported: h_res/h_post allgather not implemented."
)
class MHCLayerCommunicator(LayerCommunicator):
"""A layer whose residual is hyper-connection streams: the shared boundary
steps, run with MHCState's residual operations."""
def __init__(
self,
layer_scatter_modes: LayerScatterModes,
input_layernorm: torch.nn.Module,
post_attention_layernorm: torch.nn.Module,
allow_reduce_scatter: bool = False,
qkv_latent_func: Optional[Callable] = None,
*,
hc_mult: int,
hc_attn_pre: Callable,
hc_ffn_pre: Callable,
hc_post: Callable,
hc_ffn_post_pre: Optional[Callable] = None,
):
self.mhc = MHCState(
hc_mult=hc_mult,
hc_attn_pre=hc_attn_pre,
hc_ffn_pre=hc_ffn_pre,
hc_post=hc_post,
hc_ffn_post_pre=hc_ffn_post_pre,
is_last_layer=layer_scatter_modes.is_last_layer,
)
if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHCLayerCommunicator does not support a MoE gathered over the "
"MoE-CP group (moe_dp_size < attention_context_parallel_size). "
"Increase moe_dp_size to match attention_context_parallel_size."
)
# The postprocess writes the FFN output into the streams, so the FFN's
# sum never waits for the next layer.
super().__init__(
layer_scatter_modes,
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
qkv_latent_func,
allow_deferred_ffn_reduction=False,
residual_ops=self.mhc,
)
# MHC has not been run with input-scattered attention under attention CP.
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
raise NotImplementedError(
"MHCLayerCommunicator with input-scattered attention under attention CP"
)
def _steps_from_declarations(
self, sides: DecoderLayerSides, **kwargs
) -> BoundarySteps:
# MHC has not been run with an FFN input gathered over attention CP.
if (
TokenAxis.ATTN_CP
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
):
raise NotImplementedError(
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
)
return super()._steps_from_declarations(sides, **kwargs)
@@ -12,12 +12,13 @@ from typing import Callable, Optional, Sequence
import torch
from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis
from sglang.srt.layers.communicator import (
FfnExitFusion,
FusedMlpInput,
HandoffOutput,
LayerCommunicator,
SumGroup,
TokenAxis,
UnreducedOutput,
get_attn_tp_context,
)
+1 -1
View File
@@ -28,12 +28,12 @@ from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
MHCLayerCommunicator,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
reduce_output,
)
from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelBatchedLinear,
+6 -8
View File
@@ -3,21 +3,19 @@
from typing import Optional
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
from sglang.srt.layers.boundary_layout import (
from sglang.srt.layers.communicator import (
InputRead,
LayerCommunicator,
LayerScatterModes,
LayerStage,
Layout,
ScatterMode,
StageDecl,
StageInput,
StageOutput,
SumGroup,
TokenAxis,
stage_edges,
)
from sglang.srt.layers.communicator import (
InputRead,
LayerCommunicator,
LayerScatterModes,
LayerStage,
ScatterMode,
token_axis_sizes,
)
from sglang.srt.layers.layernorm import RMSNorm
+52
View File
@@ -0,0 +1,52 @@
"""Patching a name the communicator package's modules look up.
Each module of ``sglang.srt.layers.communicator`` binds the names it imports
in its own namespace, so a test double has to go into every module that reads
the name, not only into the package.
"""
import contextlib
from unittest import mock
from sglang.srt.layers import communicator
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
from sglang.srt.layers.communicator import residual as residual_ops
from sglang.srt.layers.communicator.adapters import attention
from sglang.srt.layers.communicator.residual import add_norm, mhc
COMMUNICATOR_MODULES = (
communicator,
layout,
output,
attention,
residual_ops,
add_norm,
mhc,
ops,
boundary,
layer,
)
@contextlib.contextmanager
def patch_communicator(name, new=mock.DEFAULT, **kwargs):
"""``mock.patch.object(module, name, new, **kwargs)`` on every communicator
module that has ``name``, all with the same replacement; with
``create=True`` and no module having it, on every module. Yields the
replacement."""
targets = [m for m in COMMUNICATOR_MODULES if hasattr(m, name)]
if not targets:
if not kwargs.get("create"):
raise AttributeError(f"no communicator module has {name!r}")
targets = list(COMMUNICATOR_MODULES)
with contextlib.ExitStack() as stack:
replacement = stack.enter_context(
mock.patch.object(targets[0], name, new, **kwargs)
)
for module in targets[1:]:
stack.enter_context(
mock.patch.object(
module, name, replacement, create=kwargs.get("create", False)
)
)
yield replacement