[Refactor] Retire the layer facade and simplify boundary internals (#41554)
This commit is contained in:
@@ -195,7 +195,10 @@ class Parallel(msgspec.Struct):
|
||||
] = None
|
||||
enable_attn_tp_input_scattered: A[
|
||||
bool,
|
||||
"Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.",
|
||||
Arg(
|
||||
help="Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.",
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
enable_shared_experts_attn_tp: A[
|
||||
bool,
|
||||
|
||||
@@ -29,6 +29,27 @@ from sglang.srt.utils.common import parse_connector_type
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
|
||||
"""Resolve the token-row modes the model's forward actually enters."""
|
||||
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
|
||||
longcat = model_type == "longcat_flash"
|
||||
if nemotron and cfg.attn_cp_size > 1:
|
||||
raise ValueError("Nemotron-H does not support --attn-cp-size > 1")
|
||||
if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1:
|
||||
raise ValueError(
|
||||
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
|
||||
)
|
||||
if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered:
|
||||
# Neither model enters the input-scattered attention scope. Preserve
|
||||
# their existing ordinary execution and make the effective flag explicit.
|
||||
logger.warning(
|
||||
"Disabling input-scattered attention for %s: its forward does not enter that scope",
|
||||
model_type,
|
||||
)
|
||||
return {"enable_attn_tp_input_scattered": False}
|
||||
return {}
|
||||
|
||||
|
||||
def handle_context_parallelism(server_args: Any):
|
||||
# Through the registry, not a bare call: an out-of-tree replacement of
|
||||
# `validate_prefill_cp_platform` registered at its own (earlier) pipeline
|
||||
@@ -42,6 +63,13 @@ def handle_context_parallelism(server_args: Any):
|
||||
model_config = model_config_of(server_args)
|
||||
hf_config = model_config.hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"boundary_parallelism",
|
||||
**_boundary_parallelism_overrides(
|
||||
cfg, model_config.hf_text_config.model_type
|
||||
),
|
||||
)
|
||||
if (
|
||||
cfg.enable_prefill_cp
|
||||
and model_arch == "DeepseekV32ForCausalLM"
|
||||
|
||||
@@ -957,7 +957,9 @@ def model_forward_stages(
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
inputs["hidden_states"] = hidden_states
|
||||
parts = _model_forward_tbo_split_inputs(
|
||||
**inputs, residual=residual, layer_input_rows=layers[0].attn_stage.input_rows
|
||||
**inputs,
|
||||
residual=residual,
|
||||
layer_input_rows=layers[0].attn_stage.incoming_residual_rows,
|
||||
)
|
||||
for part in parts:
|
||||
part["hidden_states"], child_stream = ResidualStream.arrive(
|
||||
@@ -992,89 +994,6 @@ def model_forward_stages(
|
||||
return hidden_states
|
||||
|
||||
|
||||
def model_forward_maybe_tbo(
|
||||
layers,
|
||||
enable_tbo: bool,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: Optional[BumpAllocator] = None,
|
||||
):
|
||||
inputs = dict(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
operations_strategy = OperationsStrategy.init_new_tbo(
|
||||
layers, forward_batch.global_forward_mode
|
||||
)
|
||||
if enable_tbo:
|
||||
return _model_forward_tbo(
|
||||
inputs=inputs,
|
||||
operations_strategy=operations_strategy,
|
||||
layer_input_rows=layers[0].layer_communicator.input_rows,
|
||||
)
|
||||
else:
|
||||
return _model_forward_non_tbo(inputs, operations_strategy)
|
||||
|
||||
|
||||
def _model_forward_tbo(
|
||||
inputs,
|
||||
operations_strategy: OperationsStrategy,
|
||||
layer_input_rows: Layout,
|
||||
):
|
||||
parent_batch = inputs["forward_batch"]
|
||||
stream = inputs["residual"]
|
||||
pending = stream.pending if isinstance(stream, ResidualStream) else None
|
||||
inputs["hidden_states"], inputs["residual"] = finish_layer_stack(
|
||||
inputs["hidden_states"], inputs["residual"], inputs["forward_batch"]
|
||||
)
|
||||
inputs_arr = _model_forward_tbo_split_inputs(
|
||||
**inputs, layer_input_rows=layer_input_rows
|
||||
)
|
||||
original_hidden_states_len = inputs["hidden_states"].shape[0]
|
||||
if isinstance(stream, ResidualStream):
|
||||
for part in inputs_arr:
|
||||
part["hidden_states"], part["residual"] = ResidualStream.arrive(
|
||||
part["hidden_states"],
|
||||
part["residual"],
|
||||
pending.update if pending is not None else None,
|
||||
)
|
||||
part["forward_batch"].residual_stream = part["residual"]
|
||||
del inputs, stream, pending
|
||||
|
||||
context = (
|
||||
empty_context()
|
||||
if _is_hip
|
||||
else deep_gemm_wrapper.configure_deep_gemm_num_sms(
|
||||
operations_strategy.deep_gemm_num_sms
|
||||
)
|
||||
)
|
||||
|
||||
with context:
|
||||
outputs_arr = execute_overlapped_operations(
|
||||
inputs_arr=inputs_arr,
|
||||
operations_arr=[operations_strategy.operations] * 2,
|
||||
delta_stages=[0, operations_strategy.tbo_delta_stages],
|
||||
)
|
||||
|
||||
hidden_states, residual = _model_forward_tbo_merge_outputs(
|
||||
*outputs_arr, original_hidden_states_len
|
||||
)
|
||||
parent_batch.residual_stream = (
|
||||
residual if isinstance(residual, ResidualStream) else None
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy):
|
||||
outputs = execute_operations(inputs, operations_strategy.operations)
|
||||
return outputs["hidden_states"], outputs["residual"]
|
||||
|
||||
|
||||
def _model_forward_tbo_split_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -18,24 +18,25 @@ from sglang.srt.layers.communicator.adapters.attention import (
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
LayerStage,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
stage_edges,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
BatchVariant,
|
||||
StageEdges,
|
||||
make_stage,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
HandoffRows,
|
||||
ProducerReduction,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
@@ -55,8 +56,6 @@ from sglang.srt.layers.communicator.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
@@ -78,9 +77,6 @@ from sglang.srt.layers.communicator.residual.mhc import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LayerCommunicator",
|
||||
"LayerFacts",
|
||||
"MHCLayerCommunicator",
|
||||
"declare_attn",
|
||||
"declare_ffn",
|
||||
"make_attn_stage",
|
||||
@@ -88,17 +84,16 @@ __all__ = [
|
||||
"make_stages",
|
||||
"ADD",
|
||||
"AttentionInputs",
|
||||
"BoundarySteps",
|
||||
"CommunicateSimpleFn",
|
||||
"CommunicateSummableTensorPairFn",
|
||||
"StageSteps",
|
||||
"EdgeDecl",
|
||||
"HandoffRows",
|
||||
"ProducerReduction",
|
||||
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
|
||||
"FfnCompletion",
|
||||
"FfnExit",
|
||||
"FusedMlpInput",
|
||||
"HandoffOutput",
|
||||
"LayerResidual",
|
||||
"LayerStage",
|
||||
"Layout",
|
||||
"MHCState",
|
||||
"MixerExit",
|
||||
@@ -121,14 +116,7 @@ __all__ = [
|
||||
"move_rows",
|
||||
"reduce_output",
|
||||
"sparse_moe_gathers_over_moe_cp",
|
||||
"stage_edges",
|
||||
"tbo_split_moves",
|
||||
"token_axis_sizes",
|
||||
"tp_reduce_scatter",
|
||||
]
|
||||
|
||||
from sglang.srt.layers.communicator.layer import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
)
|
||||
|
||||
@@ -135,11 +135,6 @@ class AttnTpContext:
|
||||
def set_attn_inputs(self, attn_inputs: AttentionInputs):
|
||||
get_forward().set("attn_inputs", attn_inputs)
|
||||
|
||||
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
if attn_inputs is not None:
|
||||
attn_inputs.hidden_states_local = hidden_states
|
||||
|
||||
def fetch_qkv_latent(self):
|
||||
attn_inputs = get_forward().attn_inputs
|
||||
assert attn_inputs is not None
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
from typing import TYPE_CHECKING, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -23,65 +23,63 @@ from sglang.srt.layers.communicator.ops import move_rows
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
|
||||
|
||||
def branch_input(
|
||||
self,
|
||||
source: object,
|
||||
plan: StagePlan,
|
||||
source: StagePlan,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> Tuple[torch.Tensor, ResidualStream]:
|
||||
"""The FFN input and residual that ``source``'s boundary read for its
|
||||
own FFN, for this layer's FFN, which branches from the same input: moved
|
||||
to the rows this FFN needs and its residual's rows."""
|
||||
rows, residual_rows, _ = source._branch_rows(forward_batch)
|
||||
to, residual_to, _ = self._branch_rows(forward_batch)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
raise RuntimeError("a branch must start from a prepared stage input")
|
||||
residual = stream.residual
|
||||
to, residual_to, _ = plan._branch_rows(forward_batch)
|
||||
if stream.pending is not None:
|
||||
raise RuntimeError("a branch must start from a prepared stage input")
|
||||
residual = stream.residual
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
|
||||
return (
|
||||
hidden_states,
|
||||
ResidualStream(residual) if stream is not None else residual,
|
||||
ResidualStream(residual),
|
||||
)
|
||||
|
||||
|
||||
def branch_output(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
plan: StagePlan, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
"""This layer's complete FFN output as a branch's contribution, which
|
||||
adds to the layer's output without writing the residual: moved to the
|
||||
rows the layer hands on."""
|
||||
rows, _, to = self._branch_rows(forward_batch)
|
||||
rows, _, to = plan._branch_rows(forward_batch)
|
||||
return move_rows(hidden_states, rows, to, forward_batch)
|
||||
|
||||
|
||||
def merge_branch(
|
||||
self,
|
||||
plan: StagePlan,
|
||||
contribution: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
source: object,
|
||||
stream: ResidualStream,
|
||||
source: StagePlan,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> Tuple[torch.Tensor, ResidualStream]:
|
||||
"""A contribution from ``branch_output`` summed with what ``source``'s
|
||||
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
|
||||
this layer hands on."""
|
||||
_, _, rows = source._branch_rows(forward_batch)
|
||||
_, _, to = self._branch_rows(forward_batch)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
update = None
|
||||
if stream is not None:
|
||||
stream.check(hidden_states)
|
||||
update = stream.pending.update
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
_, _, to = plan._branch_rows(forward_batch)
|
||||
stream.check(hidden_states)
|
||||
if stream.pending is None:
|
||||
raise RuntimeError("branch merge requires a pending producer contribution")
|
||||
update = stream.pending.update
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, rows, to, forward_batch)
|
||||
output = contribution + hidden_states
|
||||
if stream is not None:
|
||||
stream.write(residual)
|
||||
return stream.leave(output, update), stream
|
||||
return output, residual
|
||||
stream.write(residual)
|
||||
return stream.leave(output, update), stream
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Row requirements at the entrance to two-batch overlap."""
|
||||
|
||||
from sglang.srt.layers.communicator.contracts import HandoffRows
|
||||
from sglang.srt.layers.communicator.layout import enable_moe_dense_fully_dp
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
|
||||
def tbo_handoff(sparse, next_sparse):
|
||||
"""Record a potential split between a dense layer and a sparse layer."""
|
||||
return HandoffRows.TBO_SPLIT if not sparse and next_sparse else None
|
||||
|
||||
|
||||
def resolve_handoff_rows(requirement):
|
||||
"""Bind a handoff requirement with the construction-time parallel config.
|
||||
|
||||
Dense-local output must reach attention rows before the TBO split.
|
||||
Moving that gather after the split changes padding and collective sizes.
|
||||
"""
|
||||
if requirement is HandoffRows.TBO_SPLIT:
|
||||
return (
|
||||
HandoffRows.ATTENTION
|
||||
if enable_moe_dense_fully_dp()
|
||||
and get_exec().overlap.enable_two_batch_overlap
|
||||
else None
|
||||
)
|
||||
return requirement
|
||||
@@ -14,14 +14,18 @@
|
||||
"""What each stage produces and needs, and the boundary steps chosen from those
|
||||
declarations."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from functools import partial
|
||||
from typing import Callable, FrozenSet, Mapping, Optional, Tuple
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
CpMoves,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
@@ -32,11 +36,22 @@ from sglang.srt.layers.communicator.layout import (
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
gather_attention_tp,
|
||||
identity_output,
|
||||
move_rows,
|
||||
output_on_residual_shard,
|
||||
reduce_scatter_over_cp,
|
||||
scatter_moe_cp_output,
|
||||
scatter_output,
|
||||
take_back_attention_cp_shard,
|
||||
take_back_cp_shard,
|
||||
tp_reduce_scatter,
|
||||
tp_slice,
|
||||
update_and_gather,
|
||||
)
|
||||
from sglang.srt.layers.communicator.prepare import (
|
||||
_consumer_step,
|
||||
_dispatch_consumer,
|
||||
_hand_qkv_hook_its_input,
|
||||
_mlp_input_dp_partial,
|
||||
_mlp_input_dp_replicate,
|
||||
_mlp_input_gather_attention_cp,
|
||||
@@ -47,445 +62,10 @@ from sglang.srt.layers.communicator.ops import (
|
||||
_mlp_input_slice,
|
||||
_mlp_input_without_dp,
|
||||
_read_input,
|
||||
move_rows,
|
||||
tp_reduce_scatter,
|
||||
tp_slice,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
LayerResidual,
|
||||
StageRead,
|
||||
StageUpdate,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class StageInput(msgspec.Struct, frozen=True):
|
||||
"""The rows a stage's consumer needs: sharded over the token axes its
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
# How the consumer reads its input from the residual.
|
||||
read: StageRead = NORM_READ
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
"""What a stage's producer hands the boundary after it, fixed at
|
||||
construction: its rows, the group its output is summed over, and when it
|
||||
leaves that sum to the boundary instead of completing it.
|
||||
|
||||
What one output actually owes is carried with the value: a plain
|
||||
tensor handed across layers is complete, an ``UnreducedOutput`` names what
|
||||
is left. A raw compute output entering a boundary is read with this
|
||||
declaration and the boundary's decision for the batch."""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
group: Optional[SumGroup] = None
|
||||
# The producer never reduces its output, e.g. a row-parallel projection
|
||||
# built with reduce_results=False.
|
||||
always_leaves: bool = False
|
||||
# Otherwise it leaves the sum only when the boundary publishes the flag for
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
leaves_for_next_layer: bool = False
|
||||
leaves_for_reduce_scatter: bool = False
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
# How the producer's output is written into the residual. An arrival
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
update: Optional[StageUpdate] = ADD
|
||||
transform: Optional[OutputTransform] = None
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""What a decoder layer declares about its attention, its FFN and the rows
|
||||
between them; ``decoder_layer_edges`` turns it into the layer's
|
||||
boundaries."""
|
||||
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageDecl
|
||||
ffn: StageDecl
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
# The group the layer's input arrives as a partial sum over, if any.
|
||||
input_owes: Optional[SumGroup] = None
|
||||
# Whether the residual is added into one rank's share of the attention
|
||||
# output's sum before that sum completes, instead of after it.
|
||||
residual_joins_attention_sum: bool = False
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
rows the residual is on before and after the boundary."""
|
||||
|
||||
produced: StageOutput
|
||||
need: StageInput
|
||||
residual: Layout
|
||||
residual_to: Layout
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
# Capabilities allowed to arrive from another layer, not its update object.
|
||||
update_capabilities: Tuple[bool, ...] = ()
|
||||
|
||||
|
||||
class DecoderLayerEdges(msgspec.Struct, frozen=True):
|
||||
"""A decoder layer's three boundaries: from the rows the previous layer
|
||||
handed on into the attention, from the attention into the FFN, and from
|
||||
the FFN onto the rows the layer hands on. The next layer runs the rest of
|
||||
the last one."""
|
||||
|
||||
into_attention: EdgeDecl
|
||||
into_ffn: EdgeDecl
|
||||
out_of_ffn: EdgeDecl
|
||||
|
||||
|
||||
def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
"""The boundaries of a decoder layer, each with the declarations of its two
|
||||
sides. What arrives at the layer is complete unless it owes a sum by
|
||||
construction (``input_owes``); a sum the previous layer leaves for a batch
|
||||
comes with the value."""
|
||||
owes = sides.input_owes is not None
|
||||
# Cross-layer arrival describes rows and owed work. The actual update
|
||||
# arrives in the stream; this model family supplies only its capability.
|
||||
arrived = StageOutput(
|
||||
sides.input_rows,
|
||||
group=sides.input_owes,
|
||||
always_leaves=owes,
|
||||
update=None,
|
||||
)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
attention_rows = (
|
||||
Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER})
|
||||
if owes
|
||||
else sides.input_rows
|
||||
)
|
||||
return DecoderLayerEdges(
|
||||
into_attention=EdgeDecl(
|
||||
produced=arrived,
|
||||
need=sides.attention.input,
|
||||
residual=sides.input_rows,
|
||||
residual_to=attention_rows,
|
||||
update_capabilities=(sides.ffn.output.update.adds_plainly,),
|
||||
),
|
||||
into_ffn=EdgeDecl(
|
||||
produced=sides.attention.output,
|
||||
need=sides.ffn.input,
|
||||
residual=attention_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
),
|
||||
out_of_ffn=EdgeDecl(
|
||||
produced=sides.ffn.output,
|
||||
need=StageInput(sides.output_rows),
|
||||
residual=sides.ffn_residual_rows,
|
||||
residual_to=sides.output_rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def with_residual(
|
||||
sides: DecoderLayerSides, residual: LayerResidual
|
||||
) -> DecoderLayerSides:
|
||||
"""``sides`` with the reads and updates the layer declares for its
|
||||
attention and its FFN."""
|
||||
replace = msgspec.structs.replace
|
||||
return replace(
|
||||
sides,
|
||||
attention=StageDecl(
|
||||
input=replace(sides.attention.input, read=residual.attention_read),
|
||||
output=replace(sides.attention.output, update=residual.attention_update),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=replace(sides.ffn.input, read=residual.ffn_read),
|
||||
output=replace(sides.ffn.output, update=residual.ffn_update),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def stage_edges(
|
||||
*, previous: Optional[StageOutput], stage: StageDecl, rows: Layout
|
||||
) -> Tuple[EdgeDecl, EdgeDecl]:
|
||||
"""The two boundaries of a layer that is one stage of a sequence of stages:
|
||||
into it from the previous stage's output (None at the start of the layer
|
||||
stack) and out of it onto ``rows``, the rows every layer hands on and the
|
||||
residual is on between stages. The previous output arrives on those rows: a
|
||||
stage whose output is elsewhere (an FFN on the TP group) moves it back there
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
adds_plainly = previous.update.adds_plainly if previous is not None else True
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
update=None,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows, update=None)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
EdgeDecl(
|
||||
produced=arrived,
|
||||
need=stage.input,
|
||||
residual=rows,
|
||||
residual_to=during,
|
||||
update_capabilities=(adds_plainly,),
|
||||
),
|
||||
EdgeDecl(
|
||||
produced=stage.output,
|
||||
need=StageInput(rows),
|
||||
residual=during,
|
||||
residual_to=rows,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sequence_parallel_layer_sides(
|
||||
*, axis_sizes: Mapping[TokenAxis, int]
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer while a LayerNorm SP region is active. Its linears
|
||||
all-gather their input and reduce-scatter their output themselves, so the
|
||||
layer's rows, its residual and both outputs stay on each TP rank's slice."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
return DecoderLayerSides(
|
||||
input_rows=local,
|
||||
# qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out.
|
||||
attention=StageDecl(
|
||||
input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ),
|
||||
output=StageOutput(local),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=StageInput(
|
||||
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
|
||||
),
|
||||
output=StageOutput(local),
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=local,
|
||||
)
|
||||
|
||||
|
||||
def scattered_residual_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
is_first_layer: bool,
|
||||
is_last_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on an input-scattered batch whose residual stays on
|
||||
each attention-TP rank's slice (MHC). The attention and the FFN compute on
|
||||
the full rows. The attention takes the slice and gathers it itself (its
|
||||
QKV hook, or the handoff before an attention that needs the rows); the
|
||||
boundary brings the FFN its rows and moves both outputs back onto the
|
||||
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
|
||||
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
|
||||
is the embedding's partial sum on the full rows; the last layer hands on
|
||||
the full rows again."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
input_owes=SumGroup.TP if is_first_layer else None,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
hands_on_partial: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on a batch whose attention input is scattered over
|
||||
attention TP. Each layer's input is a partial sum over TP on the full rows:
|
||||
the embedding and every FFN before it leave that sum, and a reduce-scatter
|
||||
completes it onto each rank's slice, which the attention gathers itself.
|
||||
The residual comes back to the full rows inside the attention output's sum."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention,
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
# The next input completes the sum; the last FFN completes its own.
|
||||
output=StageOutput(
|
||||
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=attention,
|
||||
output_rows=attention,
|
||||
input_owes=SumGroup.TP,
|
||||
residual_joins_attention_sum=True,
|
||||
)
|
||||
|
||||
|
||||
def decoder_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_on_local_rows: bool,
|
||||
previous_on_local_rows: bool,
|
||||
is_last_layer: bool,
|
||||
attention_gathers_local_rows: bool,
|
||||
ffn_group: SumGroup,
|
||||
leaves_for_next_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
hands_on_attention_rows: bool = False,
|
||||
ffn_shards_over_cp: bool = False,
|
||||
) -> DecoderLayerSides:
|
||||
"""An attention followed by an FFN, derived from the groups each computes
|
||||
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
|
||||
DP shard, which completes its own combine, or a dense MLP on every rank).
|
||||
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
|
||||
computes each CP shard on its own ranks: its rows stay sharded over CP.
|
||||
A layer on local rows hands on the attention's rows when it is the last, or
|
||||
with ``hands_on_attention_rows``."""
|
||||
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
# Each attention-TP rank's own slice of those rows.
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
# The TP group spans every token axis, so an FFN on it needs every row.
|
||||
ffn = (
|
||||
local
|
||||
if ffn_on_local_rows
|
||||
else Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
)
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageDecl(
|
||||
input=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
read=NORM_QUANT_READ,
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
output=StageOutput(
|
||||
attention,
|
||||
group=SumGroup.ATTN_TP if attention_tp else None,
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
),
|
||||
ffn=StageDecl(
|
||||
input=StageInput(ffn),
|
||||
output=StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
group=ffn_group,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
),
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=(
|
||||
local
|
||||
if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows)
|
||||
else attention
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
"""A kernel that completes the sum the attention output owes together with
|
||||
the residual add and the post-attention norm, in that order.
|
||||
|
||||
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
|
||||
the residual, or None when it does not take the batch; it returns None only
|
||||
before touching its inputs or starting a collective."""
|
||||
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
"""The moves around the two-batch-overlap split, which cuts the attention's
|
||||
rows: from the rows the first overlapped layer takes to the attention's,
|
||||
@@ -493,30 +73,16 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes()
|
||||
)
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
|
||||
if layer_input_rows == attention:
|
||||
return pair._trivial, pair._trivial
|
||||
return identity_output, identity_output
|
||||
if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice: write the residual in and gather over attention
|
||||
# TP, then take the slice of each half.
|
||||
return pair._gather, pair._scatter
|
||||
return update_and_gather, scatter_output
|
||||
raise NotImplementedError(f"{layer_input_rows=}")
|
||||
|
||||
|
||||
class CpMoves(msgspec.Struct, frozen=True):
|
||||
"""How a CP extend's rows reach an FFN that needs all of them and come back,
|
||||
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
|
||||
after each rank has completed its own block, ``take_back`` returns this
|
||||
rank's block of a complete output, and ``reduce_scatter``, where there is
|
||||
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
|
||||
returns the block in the same collective."""
|
||||
|
||||
gather: Callable
|
||||
take_back: Callable
|
||||
reduce_scatter: Optional[Callable] = None
|
||||
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
|
||||
|
||||
|
||||
def _cp_moves() -> CpMoves:
|
||||
"""DSA and MLA CP gather equal shards over the attention-CP group and can
|
||||
complete a sum over it. GQA prefill CP gathers blocks padded to the longest
|
||||
@@ -524,85 +90,16 @@ def _cp_moves() -> CpMoves:
|
||||
if _gathers_over_attention_cp():
|
||||
return CpMoves(
|
||||
gather=_mlp_input_gather_attention_cp,
|
||||
take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard,
|
||||
reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp,
|
||||
take_back=take_back_attention_cp_shard,
|
||||
reduce_scatter=reduce_scatter_over_cp,
|
||||
reduce_scatter_group=lambda: get_parallel().attn_cp_group,
|
||||
)
|
||||
return CpMoves(
|
||||
gather=_mlp_input_gather_moe_cp,
|
||||
take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
||||
take_back=scatter_moe_cp_output,
|
||||
)
|
||||
|
||||
|
||||
class StageEntry(msgspec.Struct, frozen=True):
|
||||
"""The boundary into one of a layer's stages, as the layer runs it."""
|
||||
|
||||
# Completes what the input owes, writes the previous stage's output into
|
||||
# the residual and reads this stage's input:
|
||||
# (hidden_states, residual, forward_batch, norm, **call).
|
||||
prepare: Callable
|
||||
# The rows prepare hands the stage.
|
||||
input_rows: Layout
|
||||
# Moves the input onto the stage's rows after prepare, when prepare does
|
||||
# not: (hidden_states, forward_batch) -> hidden_states.
|
||||
input_move: Optional[Callable] = None
|
||||
# Hands the stage its input once it is on its rows:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
handoff: Optional[Callable] = None
|
||||
# The fused kernels prepare tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Return the updated residual to the producer's rows for aux capture.
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
preserves_residual: Optional[Callable] = None
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
input_sum: Optional[SumGroup] = None
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into each of its
|
||||
stages, and the last stage's output on to the rows the layer hands on."""
|
||||
|
||||
# The boundary into the layer's attention and into its FFN; None for a
|
||||
# stage a layer that is one stage does not have.
|
||||
attention: Optional[StageEntry]
|
||||
ffn: Optional[StageEntry]
|
||||
# What the FFN exit reads: the FFN output's group and what it may leave.
|
||||
ffn_output: StageOutput
|
||||
# The postprocess that moves the FFN output on; None when it goes back over
|
||||
# attention DP, whose step the FFN exit and postprocess choose per batch.
|
||||
ffn_output_move: Optional[Callable]
|
||||
# Whether the next layer's input can take the FFN's sum.
|
||||
ffn_sum_is_movable: bool
|
||||
# Whether ffn_output_move also completes the sum the FFN leaves.
|
||||
ffn_output_move_completes_sum: bool = False
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
return self.ffn_output_move is None
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
"""Which of a decoder layer's two stages a layer that is one stage takes
|
||||
the place of: its norm, its read and update, its fused kernels and its
|
||||
entry in the layer's steps."""
|
||||
|
||||
ATTENTION = auto()
|
||||
FFN = auto()
|
||||
|
||||
|
||||
class LayerStage(msgspec.Struct, frozen=True):
|
||||
"""A layer that is one stage of a sequence of stages, each an
|
||||
attention-like mixer or an FFN: which of the two it is, its two boundaries
|
||||
(into it, and out of it onto the rows every layer hands on, as
|
||||
``stage_edges`` gives them), and whether the layer stack
|
||||
starts at it."""
|
||||
|
||||
kind: StageKind
|
||||
edges: Tuple[EdgeDecl, EdgeDecl]
|
||||
enters_stack: bool = False
|
||||
|
||||
|
||||
class Boundary(msgspec.Struct, frozen=True):
|
||||
"""The steps one layer runs at one boundary, chosen from both sides'
|
||||
declarations. A layer runs the consumer's half of a boundary into one of
|
||||
@@ -615,24 +112,20 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
prepare: Optional[Callable] = None
|
||||
# The move onto the consumer's rows after prepare; None when there is none.
|
||||
input_move: Optional[Callable] = None
|
||||
# The fused kernels prepare tries first on a sum the input always owes.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# The producer's half: the postprocess that moves the output onto the rows
|
||||
# the layer hands on; None when it goes back over attention DP, whose step
|
||||
# the FFN exit and postprocess choose per batch.
|
||||
output_move: Optional[Callable] = None
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
returns_over_dp: bool = False
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the consumer is handed: what it needs, still sharded over
|
||||
the axes it gathers itself as the rows its input is read on are."""
|
||||
need = self.edge.need
|
||||
return Layout(
|
||||
need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself)
|
||||
)
|
||||
return input_rows(self.edge)
|
||||
|
||||
@property
|
||||
def capture_move_allocates(self) -> bool:
|
||||
@@ -648,12 +141,18 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
)
|
||||
|
||||
|
||||
def input_rows(edge: EdgeDecl) -> Layout:
|
||||
"""Consumer rows, retaining the axes it gathers inside its computation."""
|
||||
return Layout(
|
||||
edge.need.layout.sharded | (edge.residual_to.sharded & edge.need.gathers_itself)
|
||||
)
|
||||
|
||||
|
||||
def make_boundary(
|
||||
edge: EdgeDecl,
|
||||
*,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
@@ -674,7 +173,6 @@ def make_boundary(
|
||||
adds_plainly=capability,
|
||||
fusions=fusions,
|
||||
carried_fusions=carried_fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
@@ -692,9 +190,6 @@ def make_boundary(
|
||||
_dispatch_consumer,
|
||||
paths={capability: path.prepare for capability, path in paths.items()},
|
||||
),
|
||||
fused=tuple(
|
||||
f for f in fusions if any(f in path.fused for path in paths.values())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -704,7 +199,6 @@ def _bind_consumer(
|
||||
adds_plainly: bool,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
@@ -720,28 +214,26 @@ def _bind_consumer(
|
||||
producer chose for a batch."""
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
plain = adds_plainly and edge.need.read.norms_plainly
|
||||
step, fused, input_move = _select_input_steps(
|
||||
step, input_move = _select_input_steps(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions if plain else (),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
completed_step = None
|
||||
if edge.produced.always_leaves:
|
||||
completed_step, _, _ = _select_input_steps(
|
||||
completed_step, _ = _select_input_steps(
|
||||
msgspec.structs.replace(edge.produced, always_leaves=False),
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=(),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=False,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
@@ -750,14 +242,13 @@ def _bind_consumer(
|
||||
# again (e.g. MHC expansion). Both alternatives are bound at construction.
|
||||
written_step = None
|
||||
if enters_stack:
|
||||
written_step, _, _ = _select_input_steps(
|
||||
written_step, _ = _select_input_steps(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions if plain else (),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=False,
|
||||
@@ -790,7 +281,6 @@ def _bind_consumer(
|
||||
written_step=written_step,
|
||||
),
|
||||
input_move=input_move,
|
||||
fused=fused,
|
||||
)
|
||||
|
||||
|
||||
@@ -818,73 +308,15 @@ def make_output_boundary(
|
||||
residual=edge.residual,
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
update=edge.produced.update,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
output_move=None if returns_over_dp else output_move,
|
||||
output_move=output_move,
|
||||
returns_over_dp=returns_over_dp,
|
||||
output_move_completes_sum=completes_sum,
|
||||
)
|
||||
|
||||
|
||||
def _select_boundary_steps(
|
||||
sides: DecoderLayerSides,
|
||||
*,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input,
|
||||
attention_fusions: Tuple[Callable, ...] = (),
|
||||
enters_stack: bool = False,
|
||||
) -> BoundarySteps:
|
||||
"""The steps of a decoder layer: its three boundaries, each chosen from the
|
||||
declarations of its two sides, around the layer's residual operations. Both
|
||||
edges that cross attention CP take the same ``cp_moves``; the attention's
|
||||
input tries ``attention_fusions``, and the layer stack's first layer
|
||||
(``enters_stack``) starts its residual there."""
|
||||
edges = decoder_layer_edges(sides)
|
||||
out_of_ffn = make_output_boundary(edges.out_of_ffn, cp_moves=cp_moves)
|
||||
into_ffn = make_boundary(
|
||||
edges.into_ffn,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
)
|
||||
into_attention = make_boundary(
|
||||
edges.into_attention,
|
||||
carried_fusions=attention_fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention=StageEntry(
|
||||
prepare=into_attention.prepare,
|
||||
input_rows=into_attention.input_rows,
|
||||
input_move=into_attention.input_move,
|
||||
handoff=attention_handoff,
|
||||
capture_move=into_attention.capture_move,
|
||||
capture_move_allocates=into_attention.capture_move_allocates,
|
||||
input_sum=edges.into_attention.produced.group
|
||||
if edges.into_attention.produced.always_leaves
|
||||
else None,
|
||||
),
|
||||
ffn=StageEntry(
|
||||
prepare=into_ffn.prepare,
|
||||
input_rows=into_ffn.input_rows,
|
||||
input_move=into_ffn.input_move,
|
||||
fused=into_ffn.fused,
|
||||
input_sum=edges.into_ffn.produced.group
|
||||
if edges.into_ffn.produced.always_leaves
|
||||
else None,
|
||||
),
|
||||
ffn_output=edges.out_of_ffn.produced,
|
||||
ffn_output_move=out_of_ffn.output_move,
|
||||
ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum,
|
||||
ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None,
|
||||
)
|
||||
|
||||
|
||||
def _select_input_steps(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
@@ -893,11 +325,10 @@ def _select_input_steps(
|
||||
need: StageInput,
|
||||
adds_plainly: bool,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
force_layernorm_before_gather: bool,
|
||||
residual_joins_sum: bool,
|
||||
cp_moves: Optional[CpMoves],
|
||||
enters_stack: bool,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...], Optional[Callable]]:
|
||||
) -> Tuple[Callable, Optional[Callable]]:
|
||||
"""The steps into a stage for a value that is complete or owes the sum its
|
||||
producer always leaves, the fused kernels they try first, and the move
|
||||
after them: complete that sum, move the residual to the rows it has while
|
||||
@@ -933,7 +364,6 @@ def _select_input_steps(
|
||||
scatters_residual=residual != residual_to,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
@@ -946,7 +376,6 @@ def _select_input_steps(
|
||||
partial(
|
||||
_mlp_input_on_residual_shard, read=read, reduces=owes is not None
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
# A reduce-scatter completes the TP sum onto each rank's slice, which
|
||||
@@ -965,26 +394,24 @@ def _select_input_steps(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
on_chunk, fused, _ = _select_input_steps(
|
||||
on_chunk, _ = _select_input_steps(
|
||||
produced,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=StageInput(produced.layout, read=read),
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused, None
|
||||
return (partial(cp_moves.gather, gather=on_chunk), None)
|
||||
if gathered == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# A complete input on each rank's slice, gathered over attention TP
|
||||
# once it is read.
|
||||
@@ -1001,8 +428,7 @@ def _select_input_steps(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
CommunicateSimpleFn._scattered_to_tp_attn_full,
|
||||
gather_attention_tp,
|
||||
)
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
@@ -1029,7 +455,6 @@ def _select_input_steps(
|
||||
group=None,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
@@ -1039,7 +464,6 @@ def _select_input_steps(
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
if gathers_residual and residual_joins_sum:
|
||||
@@ -1047,7 +471,7 @@ def _select_input_steps(
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if owes is not SumGroup.ATTN_TP or not adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_residual_into_sum, read=read), (), None
|
||||
return (partial(_mlp_input_residual_into_sum, read=read), None)
|
||||
# A sum over TP completes only on rows every TP rank holds: the TP group
|
||||
# spans attention DP and CP.
|
||||
if owes is not SumGroup.ATTN_TP and (
|
||||
@@ -1063,7 +487,6 @@ def _select_input_steps(
|
||||
group=owes,
|
||||
read=read,
|
||||
),
|
||||
fused,
|
||||
None,
|
||||
)
|
||||
if owes not in (None, SumGroup.ATTN_TP):
|
||||
@@ -1077,7 +500,7 @@ def _select_input_steps(
|
||||
places_cp_shards = TokenAxis.ATTN_CP in gathered
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and not read.before_gather
|
||||
and adds_plainly
|
||||
and read.norms_plainly
|
||||
):
|
||||
@@ -1088,7 +511,6 @@ def _select_input_steps(
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
@@ -1099,7 +521,6 @@ def _select_input_steps(
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -1110,21 +531,21 @@ def _select_ffn_output_move(
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
update: StageUpdate = ADD,
|
||||
) -> Tuple[bool, Optional[Callable], bool]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here; and whether that move also completes the sum the FFN
|
||||
leaves."""
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
|
||||
update = produced.update
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, pair._trivial, False
|
||||
return False, identity_output, False
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return False, partial(pair._gather, update=update), False
|
||||
return False, partial(update_and_gather, update=update), False
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
|
||||
@@ -1135,7 +556,7 @@ def _select_ffn_output_move(
|
||||
return (
|
||||
False,
|
||||
partial(
|
||||
pair._onto_residual_shard,
|
||||
output_on_residual_shard,
|
||||
sums=sums,
|
||||
gathers_back=to != residual,
|
||||
update=update,
|
||||
@@ -1159,7 +580,7 @@ def _select_ffn_output_move(
|
||||
return False, cp_moves.reduce_scatter, True
|
||||
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
|
||||
# This rank's CP shard, from where the DP gather put it.
|
||||
return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False
|
||||
return False, take_back_cp_shard, False
|
||||
if returned != {TokenAxis.ATTN_DP}:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return True, None, False
|
||||
|
||||
@@ -14,22 +14,25 @@
|
||||
"""Bind one stage to predeclared boundaries and reusable batch paths."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property
|
||||
from typing import Callable, Optional, Protocol, Tuple
|
||||
from typing import Callable, Optional
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
_cp_moves,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
BatchVariant,
|
||||
CpMoves,
|
||||
EdgeDecl,
|
||||
ProducerReduction,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import OutputBoundary
|
||||
from sglang.srt.layers.communicator.fusions.allreduce import (
|
||||
attention_fusions,
|
||||
@@ -42,11 +45,14 @@ from sglang.srt.layers.communicator.layout import (
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.prepare import (
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
)
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
get_lora,
|
||||
@@ -58,24 +64,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
|
||||
|
||||
class BoundaryFusions(Protocol):
|
||||
"""The fused kernels a fusion backend gives a layer, bound to it and tried
|
||||
before the layer's own: at its attention input (each takes what the previous
|
||||
layer left, the residual, the batch and a post-residual addition), at its
|
||||
FFN input, and at its FFN exit (each says what the exit does when its kernel
|
||||
takes the batch)."""
|
||||
|
||||
def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ...
|
||||
|
||||
def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ...
|
||||
|
||||
requires_local_reduction: bool
|
||||
|
||||
def can_defer_finalize(
|
||||
self, layer: "StagePlan", forward_batch: ForwardBatch
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
"""A MoE layer under attention CP whose tokens the steps cannot bring
|
||||
to it: under a prefill CP, one dispatched per DP shard under attention DP
|
||||
@@ -118,13 +106,6 @@ def _input_can_be_scattered() -> bool:
|
||||
)
|
||||
|
||||
|
||||
class BatchVariant(Enum):
|
||||
ORDINARY = auto()
|
||||
CONTEXT_PARALLEL = auto()
|
||||
INPUT_SCATTERED = auto()
|
||||
SEQUENCE_PARALLEL = auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageEdges:
|
||||
"""One stage's incoming and outgoing declarations for one batch variant.
|
||||
@@ -136,7 +117,7 @@ class StageEdges:
|
||||
incoming: EdgeDecl
|
||||
outgoing: EdgeDecl
|
||||
handoff: Optional[Callable] = None
|
||||
cp_moves: object = None
|
||||
cp_moves: Optional[CpMoves] = None
|
||||
|
||||
|
||||
def _requires_branch_input(*args, **kwargs):
|
||||
@@ -155,26 +136,16 @@ class StagePlan:
|
||||
enters_stack=False,
|
||||
prepared_input=False,
|
||||
terminal=False,
|
||||
fixed_output=False,
|
||||
is_sparse=False,
|
||||
direct_handoff=False,
|
||||
qkv_latent_func=None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
enable_fused_ar_quant=False,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
residual_in_hidden=False,
|
||||
fusions=None,
|
||||
):
|
||||
self.kind = kind
|
||||
self.norm = norm
|
||||
self.variants = dict(variants)
|
||||
self.is_first_layer = enters_stack
|
||||
self.is_last_layer = terminal
|
||||
self.fixed_output = fixed_output
|
||||
self.is_sparse = is_sparse
|
||||
self.enters_stack = enters_stack
|
||||
self.terminal = terminal
|
||||
self.direct_handoff = direct_handoff
|
||||
self.qkv_latent_func = qkv_latent_func
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.residual_in_hidden = residual_in_hidden
|
||||
self.fusions = fusions
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
@@ -183,8 +154,13 @@ class StagePlan:
|
||||
get_lora().enable_lora
|
||||
)
|
||||
self._fusion_rows = None
|
||||
carried = attention_fusions(self) if kind is StageKind.ATTENTION else ()
|
||||
self._carried_fusions = carried
|
||||
carried = (
|
||||
attention_fusions(
|
||||
self, next(iter(self.variants.values())).incoming.need.read
|
||||
)
|
||||
if kind is StageKind.ATTENTION
|
||||
else ()
|
||||
)
|
||||
fused = ffn_fusions(self) if kind is StageKind.FFN else ()
|
||||
self._paths = {}
|
||||
for variant, edges in self.variants.items():
|
||||
@@ -200,7 +176,6 @@ class StagePlan:
|
||||
edges.incoming,
|
||||
fusions=fused,
|
||||
carried_fusions=carried,
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
cp_moves=edges.cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
@@ -209,7 +184,6 @@ class StagePlan:
|
||||
input_rows=into.input_rows,
|
||||
input_move=into.input_move,
|
||||
handoff=edges.handoff,
|
||||
fused=into.fused,
|
||||
capture_move=into.capture_move,
|
||||
capture_move_allocates=into.capture_move_allocates,
|
||||
preserves_residual=into.preserves_residual,
|
||||
@@ -219,55 +193,54 @@ class StagePlan:
|
||||
)
|
||||
out = (
|
||||
None
|
||||
if fixed_output
|
||||
if direct_handoff
|
||||
else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves)
|
||||
)
|
||||
self._paths[variant] = BoundarySteps(
|
||||
attention=entry if kind is StageKind.ATTENTION else None,
|
||||
ffn=entry if kind is StageKind.FFN else None,
|
||||
ffn_output=edges.outgoing.produced,
|
||||
ffn_output_move=None if out is None else out.output_move,
|
||||
ffn_output_move_completes_sum=False
|
||||
self._paths[variant] = StageSteps(
|
||||
entry=entry,
|
||||
output=edges.outgoing.produced,
|
||||
output_move=None if out is None else out.output_move,
|
||||
returns_over_dp=out is not None and out.returns_over_dp,
|
||||
output_move_completes_sum=False
|
||||
if out is None
|
||||
else out.output_move_completes_sum,
|
||||
ffn_sum_is_movable=edges.outgoing.produced.group is not None,
|
||||
)
|
||||
self._steps = self._paths[BatchVariant.ORDINARY]
|
||||
self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL)
|
||||
self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED)
|
||||
self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL)
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
def incoming_residual_rows(self):
|
||||
return self.variants[BatchVariant.ORDINARY].incoming.residual
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
|
||||
return TokenAxis.ATTN_TP_SCATTER in self.incoming_residual_rows.sharded
|
||||
|
||||
def _variant(self, forward_batch):
|
||||
if self._sp_steps is not None and get_forward().sp_active:
|
||||
# The batch determines its rows; missing paths must not change them.
|
||||
if get_forward().sp_active:
|
||||
return BatchVariant.SEQUENCE_PARALLEL
|
||||
if (
|
||||
self._input_scattered_steps is not None
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return BatchVariant.INPUT_SCATTERED
|
||||
if self._cp_steps is not None and _batch_shards_over_cp(forward_batch):
|
||||
if _batch_shards_over_cp(forward_batch):
|
||||
return BatchVariant.CONTEXT_PARALLEL
|
||||
return BatchVariant.ORDINARY
|
||||
|
||||
def _batch_steps(self, forward_batch):
|
||||
return self._paths[self._variant(forward_batch)]
|
||||
variant = self._variant(forward_batch)
|
||||
try:
|
||||
return self._paths[variant]
|
||||
except KeyError:
|
||||
raise NotImplementedError(
|
||||
f"no stage boundary path for the active {variant.name} batch"
|
||||
) from None
|
||||
|
||||
def fusion_rows(self, forward_batch):
|
||||
if self._fusion_rows is not None:
|
||||
return self._fusion_rows[self._variant(forward_batch)]
|
||||
entry = self._batch_steps(forward_batch)
|
||||
return (entry.attention or entry.ffn).input_rows
|
||||
return entry.entry.input_rows
|
||||
|
||||
def produced(self, kind, forward_batch):
|
||||
return self._batch_steps(forward_batch).ffn_output
|
||||
def produced(self, forward_batch):
|
||||
return self._batch_steps(forward_batch).output
|
||||
|
||||
@cached_property
|
||||
def output(self):
|
||||
@@ -284,24 +257,47 @@ class StagePlan:
|
||||
)
|
||||
|
||||
|
||||
def make_stage(
|
||||
declaration: StageDecl,
|
||||
*,
|
||||
kind: StageKind,
|
||||
norm,
|
||||
incoming: EdgeDecl,
|
||||
outgoing: EdgeDecl,
|
||||
variants=None,
|
||||
handoff=None,
|
||||
**options,
|
||||
) -> StageCommunicator:
|
||||
"""Build one real stage from its own declaration and adjacent edges.
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
variants = {}
|
||||
for variant, edge in incoming.entries.items():
|
||||
if (
|
||||
declaration.update.at_producer
|
||||
and TokenAxis.ATTN_CP
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
handoff = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
handoff = (
|
||||
_hand_scattered_input_to_attention
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and not declaration.update.adds_plainly
|
||||
else _hand_qkv_hook_its_input
|
||||
)
|
||||
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
|
||||
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
|
||||
plan = StagePlan(
|
||||
declaration.kind,
|
||||
norm,
|
||||
variants,
|
||||
enters_stack=incoming.producer is None,
|
||||
prepared_input=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
direct_handoff=declaration.kind is StageKind.ATTENTION
|
||||
and declaration.reduction is ProducerReduction.PARTIAL,
|
||||
**options,
|
||||
)
|
||||
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
|
||||
# Layout eligibility comes from the connected consumer, not a mutable
|
||||
# link to its execution plan. Kernel binding remains consumer-owned.
|
||||
from sglang.srt.layers.communicator.boundary import input_rows
|
||||
|
||||
The optional variants supply the same stage's CP/SP/input-scattered
|
||||
declarations. Neither a paired stage nor its layer facts/norm is needed.
|
||||
"""
|
||||
if incoming.need != declaration.input or outgoing.produced != declaration.output:
|
||||
raise ValueError("stage declaration disagrees with its boundaries")
|
||||
paths = dict(variants or {})
|
||||
paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff)
|
||||
return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm)
|
||||
plan._fusion_rows = {
|
||||
v: input_rows(edge) for v, edge in outgoing.entries.items()
|
||||
}
|
||||
|
||||
return StageCommunicator(plan, declaration=declaration)
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Immutable stage declarations and bound input/output paths."""
|
||||
|
||||
from enum import Enum, auto
|
||||
from typing import Callable, FrozenSet, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.layout import Layout, SumGroup, TokenAxis
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
|
||||
|
||||
class ProducerReduction(Enum):
|
||||
"""Whether compute always leaves its sum or follows the boundary scope."""
|
||||
|
||||
PARTIAL = auto()
|
||||
SCOPED = auto()
|
||||
# A replicated component is added after the sum inside compute.
|
||||
LOCAL_TAIL = auto()
|
||||
|
||||
|
||||
class HandoffRows(Enum):
|
||||
"""A required layer/branch handoff, independent of compute's input rows."""
|
||||
|
||||
ATTENTION = auto()
|
||||
TBO_SPLIT = auto()
|
||||
|
||||
|
||||
class BatchVariant(Enum):
|
||||
ORDINARY = auto()
|
||||
CONTEXT_PARALLEL = auto()
|
||||
INPUT_SCATTERED = auto()
|
||||
SEQUENCE_PARALLEL = auto()
|
||||
|
||||
|
||||
class StageInput(msgspec.Struct, frozen=True):
|
||||
"""The rows a stage's consumer needs: sharded over the token axes its
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
# How the consumer reads its input from the residual.
|
||||
read: StageRead = NORM_READ
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
"""What a stage's producer hands the boundary after it, fixed at
|
||||
construction: its rows, the group its output is summed over, and when it
|
||||
leaves that sum to the boundary instead of completing it.
|
||||
|
||||
What one output actually owes is carried with the value: a plain
|
||||
tensor handed across layers is complete, an ``UnreducedOutput`` names what
|
||||
is left. A raw compute output entering a boundary is read with this
|
||||
declaration and the boundary's decision for the batch."""
|
||||
|
||||
layout: Layout
|
||||
# None when there is nothing to sum.
|
||||
group: Optional[SumGroup] = None
|
||||
# The producer never reduces its output, e.g. a row-parallel projection
|
||||
# built with reduce_results=False.
|
||||
always_leaves: bool = False
|
||||
# Otherwise it leaves the sum only when the boundary publishes the flag for
|
||||
# it: fuse_mlp_allreduce, or mlp_reduce_scatter.
|
||||
leaves_for_next_layer: bool = False
|
||||
leaves_for_reduce_scatter: bool = False
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
# How the producer's output is written into the residual. An arrival
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
update: Optional[StageUpdate] = ADD
|
||||
transform: Optional[OutputTransform] = None
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
"""A computing stage's two sides: the rows its input must be on, and what
|
||||
its output is."""
|
||||
|
||||
input: StageInput
|
||||
output: StageOutput
|
||||
|
||||
|
||||
class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
"""One boundary between two stages, as the layer that runs one side of it
|
||||
sees it: what arrives from the producer, what the consumer needs, and the
|
||||
rows the residual is on before and after the boundary."""
|
||||
|
||||
produced: StageOutput
|
||||
need: StageInput
|
||||
residual: Layout
|
||||
residual_to: Layout
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
# Capabilities allowed to arrive from another layer, not its update object.
|
||||
update_capabilities: Tuple[bool, ...] = ()
|
||||
|
||||
|
||||
class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
"""A kernel that completes the sum the attention output owes together with
|
||||
the residual add and the post-attention norm, in that order.
|
||||
|
||||
``run(hidden_states, residual, forward_batch)`` returns the FFN's input and
|
||||
the residual, or None when it does not take the batch; it returns None only
|
||||
before touching its inputs or starting a collective."""
|
||||
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
# Callable(residual, forward_batch): True guarantees this candidate is
|
||||
# selected and does not mutate residual, including backend fallback.
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class CpMoves(msgspec.Struct, frozen=True):
|
||||
"""How a CP extend's rows reach an FFN that needs all of them and come back,
|
||||
chosen once for the kind of prefill CP: ``gather`` gathers the FFN input
|
||||
after each rank has completed its own block, ``take_back`` returns this
|
||||
rank's block of a complete output, and ``reduce_scatter``, where there is
|
||||
one, completes a sum left over the ranks of ``reduce_scatter_group()`` and
|
||||
returns the block in the same collective."""
|
||||
|
||||
gather: Callable
|
||||
take_back: Callable
|
||||
reduce_scatter: Optional[Callable] = None
|
||||
reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None
|
||||
|
||||
|
||||
class StageEntry(msgspec.Struct, frozen=True):
|
||||
"""The boundary into one of a layer's stages, as the layer runs it."""
|
||||
|
||||
# Completes what the input owes, writes the previous stage's output into
|
||||
# the residual and reads this stage's input:
|
||||
# (hidden_states, residual, forward_batch, norm, **call).
|
||||
prepare: Callable
|
||||
# The rows prepare hands the stage.
|
||||
input_rows: Layout
|
||||
# Moves the input onto the stage's rows after prepare, when prepare does
|
||||
# not: (hidden_states, forward_batch) -> hidden_states.
|
||||
input_move: Optional[Callable] = None
|
||||
# Hands the stage its input once it is on its rows:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
handoff: Optional[Callable] = None
|
||||
# Return the updated residual to the producer's rows for aux capture.
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
input_sum: Optional[SumGroup] = None
|
||||
preserves_residual: Optional[Callable] = None
|
||||
# Bound from a local successor at construction; no runtime plan lookup.
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class StageSteps(msgspec.Struct, frozen=True):
|
||||
"""One batch variant of one stage: its input and output boundary paths."""
|
||||
|
||||
entry: StageEntry
|
||||
output: StageOutput
|
||||
# None means no fixed move. returns_over_dp selects batch-dependent DP transport.
|
||||
output_move: Optional[Callable]
|
||||
output_move_completes_sum: bool = False
|
||||
|
||||
returns_over_dp: bool = False
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
"""Which of a decoder layer's two stages a layer that is one stage takes
|
||||
the place of: its norm, its read and update, its fused kernels and its
|
||||
entry in the layer's steps."""
|
||||
|
||||
ATTENTION = auto()
|
||||
FFN = auto()
|
||||
@@ -16,7 +16,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -51,11 +51,13 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
post_experts_reduction_group,
|
||||
post_experts_sum_is_one_all_reduce,
|
||||
should_use_dp_reduce_scatterv,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
post_experts_reduction_group,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
@@ -95,32 +97,31 @@ class OutputBoundary:
|
||||
def postprocess_layer(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
"""Move a complete output from the operation-scheduled producer path."""
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
hidden_states, residual = self._complete_ffn_output_now(
|
||||
hidden_states,
|
||||
stream.residual if stream is not None else residual,
|
||||
stream.residual,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=None,
|
||||
steps=steps,
|
||||
)
|
||||
return self._leave_ffn_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
stream,
|
||||
self.plan._batch_steps(forward_batch).ffn_output.update,
|
||||
steps.output.update,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
|
||||
if stream is None:
|
||||
return hidden_states, residual
|
||||
stream.residual = residual
|
||||
if update.at_producer:
|
||||
return stream.write(hidden_states), stream
|
||||
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
|
||||
return stream.write(hidden_states)
|
||||
return stream.leave(hidden_states, update, declared_sum=declared_sum)
|
||||
|
||||
@staticmethod
|
||||
def _declared_ffn_sum(steps, skipped_reduction):
|
||||
@@ -129,74 +130,69 @@ class OutputBoundary:
|
||||
if (
|
||||
skipped_reduction
|
||||
and not steps.returns_over_dp
|
||||
and not steps.ffn_output_move_completes_sum
|
||||
and not steps.output_move_completes_sum
|
||||
):
|
||||
return SumGroup.TP
|
||||
return None
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
def _local_token_move_can_go_to_next_layer(self, steps) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
the next layer's input also writes the output into the residual."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
|
||||
return steps.returns_over_dp and not steps.output.update.at_producer
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
self, forward_batch: ForwardBatch, steps
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings this layer's FFN output back to this
|
||||
rank's tokens under attention DP; None when the base postprocess would
|
||||
only scatter, or does not move tokens."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if not steps.returns_over_dp:
|
||||
return None
|
||||
return _reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
|
||||
leaves_for_reduce_scatter=steps.output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=steps.output.leaves_for_reduce_scatterv,
|
||||
)
|
||||
|
||||
def _ffn_leaves_sum_to_reduce_scatter(
|
||||
self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
|
||||
self, steps, dp_step: Optional[Callable]
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if dp_step is not None:
|
||||
return True
|
||||
if not steps.ffn_output.leaves_for_reduce_scatter:
|
||||
if not steps.output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if steps.ffn_output_move_completes_sum:
|
||||
if steps.output_move_completes_sum:
|
||||
return True
|
||||
return get_attn_tp_context().input_scattered and not self.plan.is_last_layer
|
||||
return get_attn_tp_context().input_scattered and not self.plan.terminal
|
||||
|
||||
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
|
||||
def ffn_reduction_group(self, steps) -> GroupCoordinator:
|
||||
"""The group this layer's FFN output owes its sum over: the MoE output's
|
||||
group on a sparse layer, the TP group a dense MLP reduces over."""
|
||||
return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group)
|
||||
return _sum_group(steps.output.group)
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
|
||||
def _select_ffn_completion(
|
||||
self, forward_batch: ForwardBatch, steps
|
||||
) -> FfnCompletion:
|
||||
"""Decide once, before the FFN runs, what it skips and what completes its
|
||||
output: the next layer's input, or this layer's postprocess step."""
|
||||
dp_step = self._postprocess_dp_step(forward_batch)
|
||||
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
|
||||
forward_batch, dp_step
|
||||
)
|
||||
dp_step = self._postprocess_dp_step(forward_batch, steps)
|
||||
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(steps, dp_step)
|
||||
complete_now = partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
steps=steps,
|
||||
)
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
)
|
||||
if not steps.ffn_output.leaves_for_next_layer and not (
|
||||
self.plan.is_last_layer and defer_moe_finalize
|
||||
if not steps.output.leaves_for_next_layer and not (
|
||||
self.plan.terminal and defer_moe_finalize
|
||||
):
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
@@ -207,10 +203,10 @@ class OutputBoundary:
|
||||
# Producers declare remaining work independently of the kernel chosen
|
||||
# by the consumer. Every handoff also carries an unfused completion.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
forward_batch, steps, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
group = self.ffn_reduction_group(forward_batch)
|
||||
group = self.ffn_reduction_group(steps)
|
||||
if steps.returns_over_dp:
|
||||
# Under attention DP the next layer also brings the sum back to
|
||||
# this rank's tokens.
|
||||
@@ -225,8 +221,8 @@ class OutputBoundary:
|
||||
complete = partial(_leave_to_next_layer, wrap)
|
||||
elif (
|
||||
dp_step is not None
|
||||
and not self.plan.is_last_layer
|
||||
and self._local_token_move_can_go_to_next_layer(forward_batch)
|
||||
and not self.plan.terminal
|
||||
and self._local_token_move_can_go_to_next_layer(steps)
|
||||
):
|
||||
complete = partial(
|
||||
_leave_to_next_layer,
|
||||
@@ -253,51 +249,48 @@ class OutputBoundary:
|
||||
*,
|
||||
forward_batch: ForwardBatch,
|
||||
dp_step: Optional[Callable],
|
||||
steps,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the move back to where the next layer reads the FFN output, then
|
||||
the write-back into the residual for a layer that does it itself."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if steps.ffn_output.transform is not None:
|
||||
hidden_states = steps.ffn_output.transform.apply(hidden_states)
|
||||
if steps.output.transform is not None:
|
||||
hidden_states = steps.output.transform.apply(hidden_states)
|
||||
if steps.returns_over_dp:
|
||||
hidden_states = _to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = steps.ffn_output_move(
|
||||
hidden_states, residual = steps.output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
update = steps.ffn_output.update
|
||||
update = steps.output.update
|
||||
if residual is not None and update.at_producer:
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit:
|
||||
def mixer_exit(
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
) -> MixerExit:
|
||||
"""Decide once whether this stage's mixer (an attention-like stage)
|
||||
skips its output all-reduce. Use the result as a context manager around
|
||||
the mixer, then call ``finish``."""
|
||||
return MixerExit(self, forward_batch)
|
||||
return MixerExit(self, forward_batch, stream=stream)
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit:
|
||||
def ffn_exit(
|
||||
self, forward_batch: ForwardBatch, *, stream: ResidualStream
|
||||
) -> FfnExit:
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch)
|
||||
return FfnExit(self, forward_batch, stream=stream)
|
||||
|
||||
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
|
||||
def _ffn_sum_can_move_to_next_layer(self, steps) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if (
|
||||
(
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.requires_local_reduction
|
||||
)
|
||||
or is_enable_moe_cp_allgather()
|
||||
or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable
|
||||
):
|
||||
if is_enable_moe_cp_allgather() or steps.output.group is None:
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
@@ -324,6 +317,7 @@ class OutputBoundary:
|
||||
def _ffn_sum_moves_to_next_layer(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
steps,
|
||||
*,
|
||||
mlp_reduce_scatter: bool,
|
||||
dp_step: Optional[Callable],
|
||||
@@ -333,8 +327,8 @@ class OutputBoundary:
|
||||
same all-reduce the FFN itself would have."""
|
||||
return (
|
||||
get_parallel().tp_size > 1
|
||||
and not self.plan.is_last_layer
|
||||
and self._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
and not self.plan.terminal
|
||||
and self._ffn_sum_can_move_to_next_layer(steps)
|
||||
and _can_defer_ffn_reduction(forward_batch, self.plan)
|
||||
and not mlp_reduce_scatter
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
@@ -342,7 +336,7 @@ class OutputBoundary:
|
||||
and (
|
||||
not is_dp_attention_enabled()
|
||||
or (
|
||||
self._local_token_move_can_go_to_next_layer(forward_batch)
|
||||
self._local_token_move_can_go_to_next_layer(steps)
|
||||
and dp_step is None
|
||||
)
|
||||
)
|
||||
@@ -420,16 +414,23 @@ class MixerExit:
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
|
||||
produced = communicator.plan._batch_steps(forward_batch).ffn_output
|
||||
self._stream = None
|
||||
def __init__(
|
||||
self,
|
||||
boundary: OutputBoundary,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
):
|
||||
steps = boundary.plan._batch_steps(forward_batch)
|
||||
produced = steps.output
|
||||
self._stream = stream
|
||||
self._update = produced.update
|
||||
self._declared_sum = produced.group if produced.always_leaves else None
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
and boundary._ffn_sum_can_move_to_next_layer(steps)
|
||||
)
|
||||
self.skips_reduction = produced.always_leaves or self._hands_on
|
||||
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
|
||||
@@ -441,21 +442,15 @@ class MixerExit:
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(self, hidden_states: torch.Tensor, residual=None):
|
||||
"""The mixer's output: its partial sum to the FFN stage after it, as a
|
||||
value that owes the sum to a mixer after it, or complete."""
|
||||
if self._stream is not None:
|
||||
residual = self._stream
|
||||
def finish(self, hidden_states: torch.Tensor):
|
||||
"""Hand the actual mixer result to the next boundary through the stream."""
|
||||
if self._hands_on:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
if isinstance(residual, ResidualStream):
|
||||
output = residual.leave(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
)
|
||||
return output if self._stream is not None else (output, residual)
|
||||
return hidden_states
|
||||
return self._stream.leave(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
)
|
||||
|
||||
|
||||
class FfnExit:
|
||||
@@ -464,8 +459,7 @@ class FfnExit:
|
||||
``defer_moe_finalize`` on ``get_forward()``."""
|
||||
|
||||
__slots__ = (
|
||||
"communicator",
|
||||
"forward_batch",
|
||||
"boundary",
|
||||
"defer_moe_finalize",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
@@ -476,18 +470,23 @@ class FfnExit:
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
|
||||
self._stream = None
|
||||
self.communicator = communicator
|
||||
self.forward_batch = forward_batch
|
||||
completion = communicator._select_ffn_completion(forward_batch)
|
||||
def __init__(
|
||||
self,
|
||||
boundary: OutputBoundary,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
stream: ResidualStream,
|
||||
):
|
||||
self._stream = stream
|
||||
self.boundary = boundary
|
||||
steps = boundary.plan._batch_steps(forward_batch)
|
||||
completion = boundary._select_ffn_completion(forward_batch, steps)
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
||||
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
||||
self._complete = completion.complete
|
||||
steps = communicator.plan._batch_steps(forward_batch)
|
||||
self._update = steps.ffn_output.update
|
||||
self._declared_sum = communicator._declared_ffn_sum(
|
||||
self._update = steps.output.update
|
||||
self._declared_sum = boundary._declared_ffn_sum(
|
||||
steps, completion.mlp_reduce_scatter
|
||||
)
|
||||
self._scope = get_forward().scoped(
|
||||
@@ -503,21 +502,14 @@ class FfnExit:
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
|
||||
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input, or postprocess."""
|
||||
if self._stream is not None:
|
||||
residual = self._stream
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
residual = stream.residual
|
||||
def finish(self, hidden_states: torch.Tensor):
|
||||
"""Complete or carry this output, preserving its producer update."""
|
||||
residual = self._stream.residual
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
||||
# A deferred MoE finalize handoff, consumed by the next attention input.
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
else:
|
||||
hidden_states, residual = self._complete(hidden_states, residual)
|
||||
result = self.communicator._leave_ffn_output(
|
||||
hidden_states, residual, stream, self._update, self._declared_sum
|
||||
return self.boundary._leave_ffn_output(
|
||||
hidden_states, residual, self._stream, self._update, self._declared_sum
|
||||
)
|
||||
return result[0] if self._stream is not None else result
|
||||
|
||||
@@ -9,21 +9,26 @@ from typing import Mapping, Optional
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
from sglang.srt.layers.communicator.adapters.overlap import (
|
||||
resolve_handoff_rows,
|
||||
tbo_handoff,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import _cp_moves
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
BatchVariant,
|
||||
_bind_stage,
|
||||
_input_can_be_scattered,
|
||||
_refuse_uncovered_cp_moe,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
EdgeDecl,
|
||||
HandoffRows,
|
||||
ProducerReduction,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
_cp_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
BatchVariant,
|
||||
StageEdges,
|
||||
StagePlan,
|
||||
_input_can_be_scattered,
|
||||
_refuse_uncovered_cp_moe,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
Layout,
|
||||
@@ -35,10 +40,6 @@ from sglang.srt.layers.communicator.layout import (
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
@@ -46,15 +47,12 @@ from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
)
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
|
||||
def _variants(*, ordinary_only=False):
|
||||
def _variants():
|
||||
yield BatchVariant.ORDINARY
|
||||
if ordinary_only:
|
||||
return
|
||||
if _generic_prefill_cp_shards_tokens():
|
||||
yield BatchVariant.CONTEXT_PARALLEL
|
||||
if _input_can_be_scattered():
|
||||
@@ -79,27 +77,31 @@ def _ffn_decl(
|
||||
*,
|
||||
sparse,
|
||||
terminal=False,
|
||||
next_sparse=False,
|
||||
output=None,
|
||||
output_transform=None,
|
||||
read=NORM_READ,
|
||||
update=ADD,
|
||||
local_dense=True,
|
||||
dense_tp_size=None,
|
||||
reduction=ProducerReduction.SCOPED,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
reduction = get_exec().comm.boundary_reduction
|
||||
if reduction not in ("ar", "rs", "rsv", "rs+rsv"):
|
||||
if dense_tp_size not in (None, 1, parallel.tp_size):
|
||||
raise ValueError("dense FFN rows support local compute or the full TP group")
|
||||
strategy = get_exec().comm.boundary_reduction
|
||||
if strategy not in ("ar", "rs", "rsv", "rs+rsv"):
|
||||
raise ValueError(
|
||||
"boundary_reduction must be resolved before model construction"
|
||||
)
|
||||
can_move_output = output is None or output.before_reduce_scatter
|
||||
use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output
|
||||
use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output
|
||||
can_move_output = output_transform is None or output_transform.before_reduce_scatter
|
||||
use_reduce_scatter = strategy in ("rs", "rs+rsv") and can_move_output
|
||||
use_reduce_scatterv = strategy in ("rsv", "rs+rsv") and can_move_output
|
||||
cp_shards = _generic_prefill_cp_shards_tokens()
|
||||
axes, attention, local, full = _rows(variant)
|
||||
on_local = (
|
||||
is_moe_input_scattered_across_dp_ranks()
|
||||
if sparse
|
||||
else (local_dense and enable_moe_dense_fully_dp())
|
||||
else (
|
||||
enable_moe_dense_fully_dp() if dense_tp_size is None else dense_tp_size == 1
|
||||
)
|
||||
)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_refuse_uncovered_cp_moe(on_local, cp_shards)
|
||||
@@ -118,7 +120,7 @@ def _ffn_decl(
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=read,
|
||||
),
|
||||
StageOutput(local, update=update, transform=output),
|
||||
StageOutput(local, update=update, transform=output_transform),
|
||||
),
|
||||
local,
|
||||
local,
|
||||
@@ -136,7 +138,7 @@ def _ffn_decl(
|
||||
leaves_for_reduce_scatter=use_reduce_scatter
|
||||
and (scattered_residual or not terminal),
|
||||
update=update,
|
||||
transform=output,
|
||||
transform=output_transform,
|
||||
),
|
||||
),
|
||||
residual,
|
||||
@@ -156,8 +158,8 @@ def _ffn_decl(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes
|
||||
)
|
||||
)
|
||||
output = (
|
||||
StageOutput(rows, update=update, transform=output)
|
||||
produced = (
|
||||
StageOutput(rows, update=update, transform=output_transform)
|
||||
if on_local
|
||||
else StageOutput(
|
||||
rows,
|
||||
@@ -166,22 +168,17 @@ def _ffn_decl(
|
||||
and not terminal
|
||||
and not update.at_producer
|
||||
and update.can_defer_across_layers
|
||||
and output is None,
|
||||
and output_transform is None
|
||||
and reduction is ProducerReduction.SCOPED,
|
||||
leaves_for_reduce_scatter=use_reduce_scatter and may_scatter,
|
||||
leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave,
|
||||
update=update,
|
||||
transform=output,
|
||||
transform=output_transform,
|
||||
)
|
||||
)
|
||||
gathers_for_tbo = (
|
||||
enable_moe_dense_fully_dp()
|
||||
and get_exec().overlap.enable_two_batch_overlap
|
||||
and not sparse
|
||||
and bool(next_sparse)
|
||||
)
|
||||
returned = local if on_local and not terminal and not gathers_for_tbo else attention
|
||||
returned = local if on_local and not terminal else attention
|
||||
return (
|
||||
StageDecl(StageInput(rows, read=read), output),
|
||||
StageDecl(StageInput(rows, read=read), produced),
|
||||
local if on_local else attention,
|
||||
returned,
|
||||
)
|
||||
@@ -201,13 +198,11 @@ class StageDeclaration:
|
||||
update: StageUpdate
|
||||
sparse: bool = False
|
||||
terminal: bool = False
|
||||
next_sparse: bool = False
|
||||
output: Optional[OutputTransform] = None
|
||||
ordinary_only: bool = False
|
||||
# A mixer with its own exit publishes reduction decisions before compute.
|
||||
mixer_exit: bool = False
|
||||
next_kind: Optional[StageKind] = None
|
||||
return_to_attention: bool = False
|
||||
output_transform: Optional[OutputTransform] = None
|
||||
reduction: ProducerReduction = ProducerReduction.SCOPED
|
||||
gathers_tp_input: bool = False
|
||||
dense_tp_size: Optional[int] = None
|
||||
handoff_rows: Optional[HandoffRows] = None
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -245,10 +240,11 @@ def declare_attn(
|
||||
read=NORM_QUANT_READ,
|
||||
update=ADD,
|
||||
terminal=False,
|
||||
ordinary_only=False,
|
||||
mixer_exit=False,
|
||||
next_kind=None,
|
||||
reduction=ProducerReduction.PARTIAL,
|
||||
gathers_tp_input=True,
|
||||
):
|
||||
if reduction is ProducerReduction.LOCAL_TAIL:
|
||||
raise ValueError("LOCAL_TAIL is not supported for attention stages")
|
||||
return StageDeclaration(
|
||||
StageKind.ATTENTION,
|
||||
read,
|
||||
@@ -256,9 +252,8 @@ def declare_attn(
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
ordinary_only=ordinary_only,
|
||||
mixer_exit=mixer_exit,
|
||||
next_kind=next_kind,
|
||||
reduction=reduction,
|
||||
gathers_tp_input=gathers_tp_input,
|
||||
)
|
||||
|
||||
|
||||
@@ -270,11 +265,14 @@ def declare_ffn(
|
||||
read=NORM_READ,
|
||||
update=ADD,
|
||||
terminal=False,
|
||||
output_transform=None,
|
||||
next_sparse=False,
|
||||
output=None,
|
||||
ordinary_only=False,
|
||||
return_to_attention=False,
|
||||
dense_tp_size=None,
|
||||
reduction=ProducerReduction.SCOPED,
|
||||
handoff_rows=None,
|
||||
):
|
||||
if reduction is ProducerReduction.PARTIAL:
|
||||
raise ValueError("PARTIAL is not supported for ffn stages")
|
||||
return StageDeclaration(
|
||||
StageKind.FFN,
|
||||
read,
|
||||
@@ -283,43 +281,43 @@ def declare_ffn(
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
next_sparse=next_sparse,
|
||||
output=output,
|
||||
ordinary_only=ordinary_only,
|
||||
return_to_attention=return_to_attention,
|
||||
output_transform=output_transform,
|
||||
dense_tp_size=dense_tp_size,
|
||||
reduction=reduction,
|
||||
handoff_rows=handoff_rows or tbo_handoff(sparse, next_sparse),
|
||||
)
|
||||
|
||||
|
||||
def _resolve(stage, variant):
|
||||
def _resolve(stage, variant, following=None):
|
||||
axes, attention, local, full = _rows(variant)
|
||||
if stage.update.at_producer:
|
||||
if stage.sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
)
|
||||
if stage.kind is StageKind.FFN:
|
||||
if stage.update.at_producer:
|
||||
if stage.sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
)
|
||||
declaration, residual, returned = _ffn_decl(
|
||||
variant,
|
||||
sparse=stage.sparse,
|
||||
terminal=stage.terminal,
|
||||
next_sparse=stage.next_sparse,
|
||||
output=stage.output,
|
||||
output_transform=stage.output_transform,
|
||||
read=stage.read,
|
||||
update=stage.update,
|
||||
local_dense=not stage.return_to_attention,
|
||||
dense_tp_size=stage.dense_tp_size,
|
||||
reduction=stage.reduction,
|
||||
)
|
||||
if stage.return_to_attention:
|
||||
if resolve_handoff_rows(stage.handoff_rows) is HandoffRows.ATTENTION:
|
||||
returned = attention
|
||||
return declaration, residual, returned
|
||||
sp = variant is BatchVariant.SEQUENCE_PARALLEL
|
||||
scattered = variant is BatchVariant.INPUT_SCATTERED
|
||||
gathers = (
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora)
|
||||
if stage.gathers_tp_input and (sp or scattered or _use_ag_after_qlora)
|
||||
else frozenset()
|
||||
)
|
||||
owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
@@ -329,10 +327,14 @@ def _resolve(stage, variant):
|
||||
local if sp else attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
always_leaves=owes
|
||||
and (not stage.mixer_exit or stage.next_kind is StageKind.FFN),
|
||||
and (
|
||||
stage.reduction is ProducerReduction.PARTIAL
|
||||
or (following is not None and following.kind is StageKind.FFN)
|
||||
),
|
||||
leaves_for_next_layer=owes
|
||||
and stage.mixer_exit
|
||||
and stage.next_kind is StageKind.ATTENTION,
|
||||
and stage.reduction is ProducerReduction.SCOPED
|
||||
and following is not None
|
||||
and following.kind is StageKind.ATTENTION,
|
||||
update=stage.update,
|
||||
),
|
||||
)
|
||||
@@ -351,9 +353,8 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
if producer is None and consumer is None:
|
||||
raise ValueError("a boundary needs at least one declared side")
|
||||
before, after = producer, consumer
|
||||
ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None)
|
||||
exits, entries = {}, {}
|
||||
for variant in _variants(ordinary_only=ordinary_only):
|
||||
for variant in _variants():
|
||||
_, attention, local, _ = _rows(variant)
|
||||
if before is None:
|
||||
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
|
||||
@@ -367,13 +368,16 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
residual = rows
|
||||
capabilities = (True,)
|
||||
else:
|
||||
decl, during, returned = _resolve(before, variant)
|
||||
decl, during, returned = _resolve(before, variant, following=after)
|
||||
if during is None:
|
||||
if residual_from is not None:
|
||||
if residual_from.consumer != producer:
|
||||
raise ValueError("residual source must enter the producer")
|
||||
during = residual_from.entries[variant].residual_to
|
||||
elif before.mixer_exit:
|
||||
elif (
|
||||
before.kind is StageKind.ATTENTION
|
||||
and before.reduction is ProducerReduction.SCOPED
|
||||
):
|
||||
during = attention
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -382,16 +386,24 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
exits[variant] = EdgeDecl(
|
||||
decl.output, StageInput(returned), during, returned
|
||||
)
|
||||
if before.kind is StageKind.ATTENTION and not before.mixer_exit:
|
||||
if (
|
||||
before.kind is StageKind.ATTENTION
|
||||
and before.reduction is ProducerReduction.PARTIAL
|
||||
):
|
||||
arrived, residual, capabilities = decl.output, during, ()
|
||||
else:
|
||||
owes = (
|
||||
variant is BatchVariant.INPUT_SCATTERED
|
||||
and not before.update.at_producer
|
||||
)
|
||||
carries = (before.mixer_exit or before.return_to_attention) and (
|
||||
decl.output.always_leaves or decl.output.leaves_for_next_layer
|
||||
)
|
||||
carries = (
|
||||
(
|
||||
before.kind is StageKind.ATTENTION
|
||||
and before.reduction is ProducerReduction.SCOPED
|
||||
)
|
||||
or resolve_handoff_rows(before.handoff_rows)
|
||||
is HandoffRows.ATTENTION
|
||||
) and (decl.output.always_leaves or decl.output.leaves_for_next_layer)
|
||||
arrived = StageOutput(
|
||||
returned,
|
||||
group=decl.output.group
|
||||
@@ -413,10 +425,10 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and arrived.always_leaves
|
||||
and arrived.update is None
|
||||
and not after.mixer_exit
|
||||
and after.reduction is ProducerReduction.PARTIAL
|
||||
else residual
|
||||
)
|
||||
elif after.return_to_attention:
|
||||
elif resolve_handoff_rows(after.handoff_rows) is HandoffRows.ATTENTION:
|
||||
during = (
|
||||
decl.input.layout
|
||||
if residual.sharded <= decl.input.layout.sharded
|
||||
@@ -440,7 +452,7 @@ def _connect(producer, consumer, *, residual_from=None):
|
||||
if (
|
||||
before is not None
|
||||
and before.kind is StageKind.ATTENTION
|
||||
and not before.mixer_exit
|
||||
and before.reduction is ProducerReduction.PARTIAL
|
||||
):
|
||||
exits[variant] = edge
|
||||
return StageConnection(producer, consumer, exits, entries)
|
||||
@@ -453,7 +465,11 @@ def _fork_input(prepared, consumer):
|
||||
execute the branch's input norm again.
|
||||
"""
|
||||
entries = {}
|
||||
for variant in _variants(ordinary_only=consumer.ordinary_only):
|
||||
for variant in _variants():
|
||||
if variant is not BatchVariant.ORDINARY:
|
||||
raise NotImplementedError(
|
||||
"prepared branch transport requires ordinary token rows"
|
||||
)
|
||||
source = prepared.entries[variant]
|
||||
declaration, during, _ = _resolve(consumer, variant)
|
||||
entries[variant] = EdgeDecl(
|
||||
@@ -476,7 +492,7 @@ def _incoming(stage):
|
||||
_incoming(previous)
|
||||
if previous is not None
|
||||
and previous.kind is StageKind.ATTENTION
|
||||
and not previous.mixer_exit
|
||||
and previous.reduction is ProducerReduction.PARTIAL
|
||||
else None
|
||||
)
|
||||
return _connect(
|
||||
@@ -498,78 +514,12 @@ def _connections(stage, following=None):
|
||||
return incoming, outgoing
|
||||
|
||||
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
if declaration.update.at_producer:
|
||||
if declaration.sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
)
|
||||
variants = {}
|
||||
for variant, edge in incoming.entries.items():
|
||||
if (
|
||||
declaration.update.at_producer
|
||||
and TokenAxis.ATTN_CP
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
handoff = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
handoff = (
|
||||
_hand_scattered_input_to_attention
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and not declaration.update.adds_plainly
|
||||
else _hand_qkv_hook_its_input
|
||||
)
|
||||
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
|
||||
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
|
||||
plan = StagePlan(
|
||||
declaration.kind,
|
||||
norm,
|
||||
variants,
|
||||
enters_stack=incoming.producer is None,
|
||||
prepared_input=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
fixed_output=declaration.kind is StageKind.ATTENTION
|
||||
and not declaration.mixer_exit,
|
||||
is_sparse=declaration.sparse,
|
||||
**options,
|
||||
)
|
||||
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
|
||||
# Layout eligibility comes from the connected consumer, not a mutable
|
||||
# link to its execution plan. Kernel binding remains consumer-owned.
|
||||
from sglang.srt.layers.communicator.boundary import make_boundary
|
||||
|
||||
plan._fusion_rows = {
|
||||
v: make_boundary(
|
||||
edge,
|
||||
cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None,
|
||||
force_layernorm_before_gather=options.get(
|
||||
"force_layernorm_before_dp_gather", False
|
||||
),
|
||||
).input_rows
|
||||
for v, edge in outgoing.entries.items()
|
||||
}
|
||||
return StageCommunicator(plan, declaration.kind, norm, declaration=declaration)
|
||||
|
||||
|
||||
def make_attn_stage(
|
||||
*,
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
qkv_latent_func=None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
enable_fused_ar_quant=False,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
residual_in_hidden=False,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this attention's boundaries, then bind its norm and hooks.
|
||||
@@ -587,10 +537,6 @@ def make_attn_stage(
|
||||
incoming,
|
||||
outgoing,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
|
||||
residual_in_hidden=residual_in_hidden,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
@@ -600,7 +546,6 @@ def make_ffn_stage(
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
|
||||
@@ -612,13 +557,13 @@ def make_ffn_stage(
|
||||
norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
residual_in_hidden=declaration.update.at_producer,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
|
||||
def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
|
||||
def make_stages(
|
||||
*stages, previous=None, prepared_from=None, following=None, terminal=False
|
||||
):
|
||||
"""Bind a local linear sequence and return its independent boundaries.
|
||||
|
||||
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
|
||||
@@ -653,53 +598,43 @@ def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
|
||||
previous=previous if index == 0 else declarations[-1],
|
||||
prepared_from=prepared_from if index == 0 else None,
|
||||
terminal=terminal and index == len(stages) - 1,
|
||||
next_kind=stages[index + 1][0].kind
|
||||
if declaration.mixer_exit and index + 1 < len(stages)
|
||||
else declaration.next_kind,
|
||||
)
|
||||
declarations.append(declaration)
|
||||
bindings.append((norm, item[2] if len(item) == 3 else {}))
|
||||
external_consumer = following
|
||||
boundaries = []
|
||||
incoming = _incoming(declarations[0])
|
||||
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
|
||||
build = (
|
||||
make_attn_stage
|
||||
if declaration.kind is StageKind.ATTENTION
|
||||
else make_ffn_stage
|
||||
)
|
||||
boundaries.append(
|
||||
build(
|
||||
declaration=declaration,
|
||||
norm=norm,
|
||||
following=declarations[index + 1]
|
||||
if index + 1 < len(declarations)
|
||||
else None,
|
||||
**options,
|
||||
)
|
||||
following = (
|
||||
declarations[index + 1]
|
||||
if index + 1 < len(declarations)
|
||||
else external_consumer
|
||||
)
|
||||
outgoing = _connect(declaration, following, residual_from=incoming)
|
||||
options = dict(options)
|
||||
if declaration.kind is StageKind.FFN:
|
||||
unexpected = options.keys() - {
|
||||
"fusions",
|
||||
}
|
||||
if unexpected:
|
||||
raise TypeError(f"unsupported FFN options: {sorted(unexpected)}")
|
||||
boundaries.append(_bind_stage(declaration, norm, incoming, outgoing, **options))
|
||||
incoming = outgoing
|
||||
for producer, consumer in zip(boundaries, boundaries[1:]):
|
||||
if (
|
||||
producer.kind is not StageKind.ATTENTION
|
||||
or producer.declaration.mixer_exit
|
||||
or consumer.kind is not StageKind.FFN
|
||||
if not (
|
||||
producer.kind is StageKind.ATTENTION
|
||||
and producer.declaration.reduction is ProducerReduction.PARTIAL
|
||||
and consumer.kind is StageKind.FFN
|
||||
):
|
||||
continue
|
||||
for variant, steps in producer.plan._paths.items():
|
||||
predicate = consumer.plan._paths[variant].ffn.preserves_residual
|
||||
next_steps = consumer.plan._paths[variant]
|
||||
predicate = next_steps.entry.preserves_residual
|
||||
if predicate is not None:
|
||||
producer.plan._paths[variant] = msgspec.structs.replace(
|
||||
steps,
|
||||
attention=msgspec.structs.replace(
|
||||
steps.attention, capture_preserves_residual=predicate
|
||||
entry=msgspec.structs.replace(
|
||||
steps.entry, capture_preserves_residual=predicate
|
||||
),
|
||||
)
|
||||
producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY]
|
||||
producer.plan._cp_steps = producer.plan._paths.get(
|
||||
BatchVariant.CONTEXT_PARALLEL
|
||||
)
|
||||
producer.plan._input_scattered_steps = producer.plan._paths.get(
|
||||
BatchVariant.INPUT_SCATTERED
|
||||
)
|
||||
producer.plan._sp_steps = producer.plan._paths.get(
|
||||
BatchVariant.SEQUENCE_PARALLEL
|
||||
)
|
||||
return tuple(boundaries)
|
||||
|
||||
@@ -19,23 +19,26 @@ from typing import Callable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.boundary import FusedMlpInput
|
||||
from sglang.srt.layers.communicator.contracts import FusedMlpInput
|
||||
from sglang.srt.layers.communicator.layout import SumGroup
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
NORM_QUANT_READ,
|
||||
Fp8Input,
|
||||
NormQuantRead,
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import post_experts_reduction_group
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
|
||||
|
||||
def attention_fusions(plan) -> Tuple[Callable, ...]:
|
||||
def attention_fusions(plan, read=NORM_QUANT_READ) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete what the previous layer left together
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
@@ -45,12 +48,23 @@ def attention_fusions(plan) -> Tuple[Callable, ...]:
|
||||
given = plan.fusions.attention_input(plan) if plan.fusions else ()
|
||||
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
|
||||
return given
|
||||
plan._attn_input_fuses_quant = (
|
||||
plan.enable_fused_ar_quant
|
||||
fuses_quant = (
|
||||
isinstance(read, NormQuantRead)
|
||||
and read.fp8_input is not None
|
||||
and _use_aiter
|
||||
and not get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false")
|
||||
and get_exec().comm.enable_aiter_allreduce_fusion
|
||||
and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group")
|
||||
)
|
||||
return (*given, partial(complete_attention_input, plan))
|
||||
return (
|
||||
*given,
|
||||
partial(
|
||||
complete_attention_input,
|
||||
plan,
|
||||
fuses_quant=fuses_quant,
|
||||
keep_bf16=fuses_quant and read.fp8_input is Fp8Input.TUPLE_AND_BF16,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def complete_attention_input(
|
||||
@@ -59,11 +73,14 @@ def complete_attention_input(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
post_residual_addition: Optional[torch.Tensor],
|
||||
*,
|
||||
fuses_quant: bool,
|
||||
keep_bf16: bool,
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Complete the sum the previous layer left, add it to the residual and
|
||||
apply the input norm in one aiter or flashinfer kernel; None when the
|
||||
kernel does not take this batch. The result is not quantized for
|
||||
``quant_format``."""
|
||||
kernel does not take this batch. The optional FP8 result follows the consumer read declaration;
|
||||
the separate ``quant_format`` path remains the read adapter's responsibility."""
|
||||
if (
|
||||
not isinstance(owed, UnreducedOutput)
|
||||
# The kernel does not add it.
|
||||
@@ -78,14 +95,14 @@ def complete_attention_input(
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
):
|
||||
return None
|
||||
if plan._attn_input_fuses_quant:
|
||||
if fuses_quant:
|
||||
# Falls back to AR+RMSNorm + separate quant internally when the
|
||||
# fully-fused kernel cannot service the shape.
|
||||
quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group(
|
||||
hidden_states,
|
||||
residual,
|
||||
use_attn_tp_group=False,
|
||||
keep_bf16=plan.fused_ar_quant_keep_bf16,
|
||||
keep_bf16=keep_bf16,
|
||||
)
|
||||
if quant_result is not None:
|
||||
return quant_result
|
||||
|
||||
@@ -208,21 +208,18 @@ class CuteDSLFusion:
|
||||
# Producer capabilities; consumer kernel selection is independent.
|
||||
self.hands_off_finalize = False
|
||||
self.terminal_finalize = False
|
||||
self.requires_local_reduction = False
|
||||
|
||||
def install(
|
||||
self,
|
||||
service: CuteDSLFusionService,
|
||||
*,
|
||||
hands_off_finalize: bool,
|
||||
output_is_replicated: bool,
|
||||
terminal_finalize: bool = False,
|
||||
) -> None:
|
||||
"""Install the shared workspace and this producer's capabilities."""
|
||||
self.service = service
|
||||
self.hands_off_finalize = hands_off_finalize
|
||||
self.terminal_finalize = terminal_finalize
|
||||
self.requires_local_reduction = output_is_replicated
|
||||
|
||||
def attention_input(self, layer) -> tuple:
|
||||
return (
|
||||
@@ -233,7 +230,7 @@ class CuteDSLFusion:
|
||||
def ffn_input(self, layer) -> tuple:
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
TokenAxis.ATTN_TP_SCATTER not in layer.input_rows.sharded
|
||||
TokenAxis.ATTN_TP_SCATTER not in layer.incoming_residual_rows.sharded
|
||||
# The workspace sums over TP, which is then the attention-TP group.
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(layer.norm) is not None
|
||||
@@ -336,11 +333,7 @@ class CuteDSLFusion:
|
||||
"""Whether this producer can emit a handoff with an unfused fallback."""
|
||||
return (
|
||||
self.hands_off_finalize
|
||||
and (not layer.is_last_layer or self.terminal_finalize)
|
||||
and (
|
||||
not self.requires_local_reduction
|
||||
or (layer.is_last_layer and self.terminal_finalize)
|
||||
)
|
||||
and (not layer.terminal or self.terminal_finalize)
|
||||
and self._should_use_finalize(
|
||||
layer, forward_batch, int(forward_batch.input_ids.shape[0])
|
||||
)
|
||||
@@ -377,7 +370,6 @@ def install_cutedsl_fusion(
|
||||
top_k: int,
|
||||
rms_epsilon: float,
|
||||
can_defer_finalize: _LayerPredicate,
|
||||
requires_local_reduction: _LayerPredicate | None = None,
|
||||
label: str,
|
||||
terminal_finalize: bool = False,
|
||||
) -> CuteDSLFusionService | None:
|
||||
@@ -424,8 +416,6 @@ def install_cutedsl_fusion(
|
||||
service,
|
||||
hands_off_finalize=hands_off_finalize,
|
||||
terminal_finalize=terminal_finalize,
|
||||
output_is_replicated=requires_local_reduction is not None
|
||||
and bool(requires_local_reduction(layer)),
|
||||
)
|
||||
logger.info(
|
||||
"Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers "
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -142,7 +142,9 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
return parallel.attn_tp_group
|
||||
if group is SumGroup.TP:
|
||||
return parallel.tp_group
|
||||
return post_experts_reduction_group()
|
||||
if group is SumGroup.MOE_OUTPUT:
|
||||
return post_experts_reduction_group()
|
||||
raise ValueError(f"unsupported output sum group: {group!r}")
|
||||
|
||||
|
||||
def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]:
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A stage's model-facing input and output boundary."""
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.communicator.residual.batch import current
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.layer import LayerCommunicator
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
|
||||
self._layer = layer
|
||||
self._name = name
|
||||
self.norm = getattr(layer, norm_name)
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
entry = getattr(steps, self._name)
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self._name} stage"
|
||||
)
|
||||
return entry
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
stream = current(forward_batch)
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
):
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
call["pending"] = stream.pending
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, **call
|
||||
)
|
||||
if entry.input_move is not None:
|
||||
hidden_states = entry.input_move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if entry.handoff is not None:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self._layer.qkv_latent_func
|
||||
)
|
||||
stream.write(residual)
|
||||
return hidden_states, stream
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
return self._layer.input_rows
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
return self._layer.input_on_attention_tp_slices
|
||||
|
||||
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
|
||||
stream = current(forward_batch)
|
||||
if self._name == "attention":
|
||||
hidden_states, stream = (
|
||||
self._layer.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states, stream, forward_batch, **call
|
||||
)
|
||||
)
|
||||
else:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
self._layer.publish_mlp_lora_layout(steps)
|
||||
if cache is not None:
|
||||
call["cache"] = cache
|
||||
hidden_states, stream = self._prepare(
|
||||
hidden_states, stream, forward_batch, steps, **call
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
"""Hand this attention's actual contribution to the following stage."""
|
||||
if self._name != "attention" or self._layer.stage_edges is not None:
|
||||
raise RuntimeError(
|
||||
"use the stage exit scope for an FFN or single-stage mixer"
|
||||
)
|
||||
return current(forward_batch).leave(
|
||||
hidden_states,
|
||||
self._layer._residual.attention_update,
|
||||
declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum,
|
||||
)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
result = (
|
||||
self._layer.ffn_exit(forward_batch)
|
||||
if self._name == "ffn"
|
||||
else self._layer.mixer_exit(forward_batch)
|
||||
)
|
||||
result._stream = current(forward_batch)
|
||||
return result
|
||||
|
||||
def from_pp(self, tensors, forward_batch, **kwargs):
|
||||
hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def snapshot(self, hidden_states, forward_batch, *, at_input=False):
|
||||
return self._layer.snapshot(
|
||||
hidden_states, current(forward_batch), at_input=at_input
|
||||
)
|
||||
|
||||
def capture_output(self, hidden_states, forward_batch, **kwargs):
|
||||
return self._layer.capture_output(
|
||||
hidden_states, current(forward_batch), **kwargs
|
||||
)
|
||||
|
||||
def postprocess(self, hidden_states, forward_batch):
|
||||
hidden_states, stream = self._layer.postprocess_layer(
|
||||
hidden_states, current(forward_batch), forward_batch
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
@@ -22,36 +22,25 @@ from sglang.srt.distributed import (
|
||||
GroupCoordinator,
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
attention_tensor_model_parallel_quant_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
|
||||
from sglang.srt.layers.communicator.residual import StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
@@ -63,21 +52,11 @@ from sglang.srt.layers.dp_attention import (
|
||||
get_global_dp_buffer,
|
||||
get_local_dp_buffer,
|
||||
get_moe_cp_rank,
|
||||
get_moe_cp_size,
|
||||
is_allocation_symmetric,
|
||||
moe_cp_all_gather_into_tensor,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
is_npu,
|
||||
)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
||||
|
||||
|
||||
def tp_reduce_scatter(
|
||||
@@ -110,43 +89,34 @@ def tp_slice(hidden_states, residual):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class CommunicateSimpleFn:
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
@staticmethod
|
||||
def _scattered_to_tp_attn_full(
|
||||
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
parallel = get_parallel()
|
||||
if isinstance(hidden_states, tuple):
|
||||
gathered_hidden_states = []
|
||||
for local_hidden_states in hidden_states:
|
||||
with use_symmetric_memory(
|
||||
parallel.tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
output = torch.empty(
|
||||
(
|
||||
local_hidden_states.shape[0] * parallel.attn_tp_size,
|
||||
*local_hidden_states.shape[1:],
|
||||
),
|
||||
dtype=local_hidden_states.dtype,
|
||||
device=local_hidden_states.device,
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(
|
||||
output,
|
||||
local_hidden_states,
|
||||
def gather_attention_tp(
|
||||
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
parallel = get_parallel()
|
||||
if isinstance(hidden_states, tuple):
|
||||
gathered_hidden_states = []
|
||||
for local_hidden_states in hidden_states:
|
||||
with use_symmetric_memory(
|
||||
parallel.tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
output = torch.empty(
|
||||
(
|
||||
local_hidden_states.shape[0] * parallel.attn_tp_size,
|
||||
*local_hidden_states.shape[1:],
|
||||
),
|
||||
dtype=local_hidden_states.dtype,
|
||||
device=local_hidden_states.device,
|
||||
)
|
||||
gathered_hidden_states.append(output)
|
||||
return tuple(gathered_hidden_states)
|
||||
attn_tp_all_gather_into_tensor(
|
||||
output,
|
||||
local_hidden_states,
|
||||
)
|
||||
gathered_hidden_states.append(output)
|
||||
return tuple(gathered_hidden_states)
|
||||
|
||||
return _redistribute_from_attn_tp_shards(hidden_states)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states)
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
@@ -244,145 +214,6 @@ def move_rows(
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _mlp_input_without_dp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Complete the sum the input owes over ``group`` on the rows it is on,
|
||||
unless one of ``fusions`` does it with the residual add and the norm, then
|
||||
write it into the residual and read the input."""
|
||||
if gathers_residual:
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
if group is SumGroup.ATTN_TP:
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, cache)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
|
||||
residual,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_dp_partial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then read
|
||||
the input from the sum. With ``places_cp_shards`` each CP rank puts its
|
||||
shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
if get_parallel().attn_tp_rank == 0:
|
||||
hidden_states += residual
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
hidden_states = _reduce_and_redistribute_output_to_dp(
|
||||
hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
|
||||
hidden_states, _ = read.read(hidden_states, layernorm)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _hand_qkv_hook_its_input(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Give the attention's QKV hook the attention input."""
|
||||
if qkv_latent_func is not None:
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _hand_scattered_input_to_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Input-scattered attention takes each rank's slice, and its QKV hook
|
||||
gathers the rows after the projection. DSA and attention without a hook
|
||||
consume full hidden states, so those are gathered here, and the hook is
|
||||
told they are."""
|
||||
ctx = get_attn_tp_context()
|
||||
if ctx.is_dsa or qkv_latent_func is None:
|
||||
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
|
||||
if qkv_latent_func is not None:
|
||||
ctx.set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
qkv_latent_func,
|
||||
is_pre_gathered=ctx.is_dsa,
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _tp_all_gather_scattered_rows(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
@@ -393,267 +224,6 @@ def _tp_all_gather_scattered_rows(
|
||||
return output
|
||||
|
||||
|
||||
def _dispatch_consumer(
|
||||
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
|
||||
):
|
||||
if residual is None:
|
||||
# Enter/read-only paths do not execute an update.
|
||||
prepare = paths[False]
|
||||
else:
|
||||
try:
|
||||
prepare = paths[update.adds_plainly]
|
||||
except KeyError:
|
||||
raise RuntimeError("producer update has no bound input path") from None
|
||||
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _consumer_step(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
*,
|
||||
step: Callable,
|
||||
adds_plainly: bool,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
expected_sum: Optional[SumGroup] = None,
|
||||
completed_step: Optional[Callable] = None,
|
||||
pending=None,
|
||||
written_step: Optional[Callable] = None,
|
||||
written: bool = False,
|
||||
update: StageUpdate = ADD,
|
||||
**call,
|
||||
):
|
||||
"""A boundary's half into a stage: complete what the value carries, a sum
|
||||
or a handoff its producer left for this batch, then run ``step`` on the
|
||||
complete value. Under attention DP the reduction back to this rank's tokens
|
||||
comes first; otherwise one of ``carried_fusions`` may complete it with the
|
||||
residual add and the read, and an empty batch has nothing to sum. An input
|
||||
whose declared sum is already completed uses ``completed_step`` instead;
|
||||
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if residual is not None and update.adds_plainly != adds_plainly:
|
||||
raise RuntimeError("producer update does not match the boundary's capability")
|
||||
if pending is not None:
|
||||
if isinstance(pending.owed, DeclaredSum):
|
||||
if pending.owed.group is not expected_sum:
|
||||
raise RuntimeError("producer sum does not match the input declaration")
|
||||
elif expected_sum is not None:
|
||||
if pending.owed is not None:
|
||||
raise RuntimeError("a declared input sum carried another completion")
|
||||
step = completed_step
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if expected_sum is not None:
|
||||
raise RuntimeError(
|
||||
f"an input that owes its sum by construction arrived as "
|
||||
f"{type(owed).__name__}"
|
||||
)
|
||||
if (
|
||||
isinstance(owed, UnreducedOutput)
|
||||
and owed.reduce_and_redistribute is not None
|
||||
):
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
for fused in carried_fusions:
|
||||
result = fused(
|
||||
owed, residual, forward_batch, call.get("post_residual_addition")
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _read_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
layer_input: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
update: StageUpdate = ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Complete what the input owes by construction (``layer_input``), then
|
||||
write the previous stage's output into the residual and read the stage's
|
||||
input with ``norm``. The layer stack's first stage (``enters_stack``)
|
||||
starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual)
|
||||
if enters:
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
return read.update_and_read(
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
|
||||
if scatters_residual:
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_slice(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
reduces: bool = True,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
if reduces:
|
||||
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
else:
|
||||
shard = shard.clone()
|
||||
shard, residual = read.update_and_read(update, shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_residual_into_sum(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, read
|
||||
)
|
||||
|
||||
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
scattered_states += residual
|
||||
residual = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return read.read(residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_gather_attention_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
return dsa_cp_gather_hidden_states(hidden_states), residual
|
||||
|
||||
|
||||
def _mlp_input_gather_moe_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
||||
TP_ATTN_FULL."""
|
||||
# Early return on empty tensor is safe for MOE_CP because:
|
||||
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
||||
# so no rank hits this path while others proceed to the allgather.
|
||||
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
||||
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None and hidden_states.shape[0] > 0:
|
||||
hidden_states = _redistribute_input_to_moe_cp(
|
||||
hidden_states, rows, get_moe_cp_size()
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _dp_scatter_group() -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if parallel.tp_size == parallel.attn_dp_size:
|
||||
@@ -720,140 +290,134 @@ def _all_reduce_then_to_local_tokens(
|
||||
)
|
||||
|
||||
|
||||
class CommunicateSummableTensorPairFn:
|
||||
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
|
||||
def identity_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
def update_and_gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
update: StageUpdate = ADD,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
|
||||
def output_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
update: StageUpdate,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
parallel = get_parallel()
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = update.update(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
update: StageUpdate = ADD,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _onto_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
update: StageUpdate,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
parallel = get_parallel()
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
|
||||
parallel.tp_rank
|
||||
]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = update.update(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
|
||||
def scatter_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
assert residual is None, "not yet handled residual!=None"
|
||||
return _redistribute_to_attn_tp_shards(hidden_states), None
|
||||
|
||||
|
||||
def take_back_attention_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: this rank's shard of a complete output gathered in
|
||||
equal shards over the attention-CP group."""
|
||||
parallel = get_parallel()
|
||||
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
|
||||
return shard, residual
|
||||
|
||||
|
||||
def reduce_scatter_over_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
keep this rank's shard."""
|
||||
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
|
||||
|
||||
|
||||
def take_back_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""This rank's CP shard of the rows the DP gather put in its DP group's
|
||||
slot, at the shard's padded length with the padding zeroed."""
|
||||
held = forward_batch.attn_cp_metadata.per_rank_actual_token
|
||||
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
|
||||
: held[get_parallel().attn_cp_rank]
|
||||
]
|
||||
dp_scatter(
|
||||
local_hidden_states,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
_cp_shard_token_rows(forward_batch),
|
||||
)
|
||||
return local_hidden_states, residual
|
||||
|
||||
|
||||
def scatter_moe_cp_output(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
||||
|
||||
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
||||
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
||||
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
||||
CP-local portion.
|
||||
|
||||
If DP>1, further scatter back to the local DP slice.
|
||||
"""
|
||||
# Only scatter back during prefill; decode was never allgathered so no-op.
|
||||
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
|
||||
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None:
|
||||
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
|
||||
|
||||
if get_parallel().attn_dp_size > 1:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
assert residual is None, "not yet handled residual!=None"
|
||||
return _redistribute_to_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _take_back_attention_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: this rank's shard of a complete output gathered in
|
||||
equal shards over the attention-CP group."""
|
||||
parallel = get_parallel()
|
||||
shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank]
|
||||
return shard, residual
|
||||
|
||||
@staticmethod
|
||||
def _reduce_scatter_over_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
keep this rank's shard."""
|
||||
return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual
|
||||
|
||||
@staticmethod
|
||||
def _take_back_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""This rank's CP shard of the rows the DP gather put in its DP group's
|
||||
slot, at the shard's padded length with the padding zeroed."""
|
||||
held = forward_batch.attn_cp_metadata.per_rank_actual_token
|
||||
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[
|
||||
: held[get_parallel().attn_cp_rank]
|
||||
]
|
||||
dp_scatter(
|
||||
local_hidden_states,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
_cp_shard_token_rows(forward_batch),
|
||||
)
|
||||
return local_hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs,
|
||||
):
|
||||
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
||||
|
||||
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
||||
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
||||
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
||||
CP-local portion.
|
||||
|
||||
If DP>1, further scatter back to the local DP slice.
|
||||
"""
|
||||
# Only scatter back during prefill; decode was never allgathered so no-op.
|
||||
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
|
||||
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None:
|
||||
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
|
||||
|
||||
if get_parallel().attn_dp_size > 1:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Complete pending output, update the residual, and read the consumer input."""
|
||||
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
_redistribute_from_attn_tp_shards,
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
HandoffOutput,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
dp_scatter,
|
||||
get_moe_cp_size,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import (
|
||||
is_npu,
|
||||
)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
||||
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
_mlp_input_reduce_output,
|
||||
_redistribute_input_to_dp,
|
||||
_redistribute_input_to_moe_cp,
|
||||
_reduce_and_redistribute_output_to_attn_tp_shards,
|
||||
_reduce_and_redistribute_output_to_dp,
|
||||
_tp_all_gather_scattered_rows,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_without_dp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Complete the sum the input owes over ``group`` on the rows it is on,
|
||||
unless one of ``fusions`` does it with the residual add and the norm, then
|
||||
write it into the residual and read the input."""
|
||||
if gathers_residual:
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
if group is SumGroup.ATTN_TP:
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=update.adds_plainly
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, cache)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = update.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = read.update_and_read(
|
||||
update, hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
|
||||
residual,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_dp_partial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then read
|
||||
the input from the sum. With ``places_cp_shards`` each CP rank puts its
|
||||
shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
if get_parallel().attn_tp_rank == 0:
|
||||
hidden_states += residual
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
hidden_states = _reduce_and_redistribute_output_to_dp(
|
||||
hidden_states, forward_batch, cp_shard_counts
|
||||
)
|
||||
dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts)
|
||||
hidden_states, _ = read.read(hidden_states, layernorm)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _hand_qkv_hook_its_input(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Give the attention's QKV hook the attention input."""
|
||||
if qkv_latent_func is not None:
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _hand_scattered_input_to_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Input-scattered attention takes each rank's slice, and its QKV hook
|
||||
gathers the rows after the projection. DSA and attention without a hook
|
||||
consume full hidden states, so those are gathered here, and the hook is
|
||||
told they are."""
|
||||
ctx = get_attn_tp_context()
|
||||
if ctx.is_dsa or qkv_latent_func is None:
|
||||
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
|
||||
if qkv_latent_func is not None:
|
||||
ctx.set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
qkv_latent_func,
|
||||
is_pre_gathered=ctx.is_dsa,
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _dispatch_consumer(
|
||||
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
|
||||
):
|
||||
if residual is None:
|
||||
# A missing residual cannot join a partial sum. The non-plain path
|
||||
# performs enter/read without that optimization.
|
||||
prepare = paths[False]
|
||||
else:
|
||||
try:
|
||||
prepare = paths[update.adds_plainly]
|
||||
except KeyError:
|
||||
raise RuntimeError("producer update has no bound input path") from None
|
||||
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _consumer_step(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
*,
|
||||
step: Callable,
|
||||
adds_plainly: bool,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
expected_sum: Optional[SumGroup] = None,
|
||||
completed_step: Optional[Callable] = None,
|
||||
pending=None,
|
||||
written_step: Optional[Callable] = None,
|
||||
written: bool = False,
|
||||
update: StageUpdate = ADD,
|
||||
**call,
|
||||
):
|
||||
"""A boundary's half into a stage: complete what the value carries, a sum
|
||||
or a handoff its producer left for this batch, then run ``step`` on the
|
||||
complete value. Under attention DP the reduction back to this rank's tokens
|
||||
comes first; otherwise one of ``carried_fusions`` may complete it with the
|
||||
residual add and the read, and an empty batch has nothing to sum. An input
|
||||
whose declared sum is already completed uses ``completed_step`` instead;
|
||||
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if residual is not None and update.adds_plainly != adds_plainly:
|
||||
raise RuntimeError("producer update does not match the boundary's capability")
|
||||
if pending is not None:
|
||||
if isinstance(pending.owed, DeclaredSum):
|
||||
if pending.owed.group is not expected_sum:
|
||||
raise RuntimeError("producer sum does not match the input declaration")
|
||||
elif expected_sum is not None:
|
||||
if pending.owed is not None:
|
||||
raise RuntimeError("a declared input sum carried another completion")
|
||||
step = completed_step
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
||||
if expected_sum is not None:
|
||||
raise RuntimeError(
|
||||
f"an input that owes its sum by construction arrived as "
|
||||
f"{type(owed).__name__}"
|
||||
)
|
||||
if (
|
||||
isinstance(owed, UnreducedOutput)
|
||||
and owed.reduce_and_redistribute is not None
|
||||
):
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
for fused in carried_fusions:
|
||||
result = fused(
|
||||
owed, residual, forward_batch, call.get("post_residual_addition")
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
if isinstance(owed, HandoffOutput) or owed.partial.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _read_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
layer_input: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
update: StageUpdate = ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Complete what the input owes by construction (``layer_input``), then
|
||||
write the previous stage's output into the residual and read the stage's
|
||||
input with ``norm``. The layer stack's first stage (``enters_stack``)
|
||||
starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual)
|
||||
if enters:
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
return read.read(hidden_states, norm, quant_format)
|
||||
return read.update_and_read(
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
|
||||
if scatters_residual:
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_slice(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
reduces: bool = True,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
if reduces:
|
||||
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
else:
|
||||
shard = shard.clone()
|
||||
shard, residual = read.update_and_read(update, shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _mlp_input_residual_into_sum(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, read
|
||||
)
|
||||
|
||||
|
||||
def _mlp_input_gather_attention_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
return dsa_cp_gather_hidden_states(hidden_states), residual
|
||||
|
||||
|
||||
def _mlp_input_gather_moe_cp(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
||||
TP_ATTN_FULL."""
|
||||
# Early return on empty tensor is safe for MOE_CP because:
|
||||
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
||||
# so no rank hits this path while others proceed to the allgather.
|
||||
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
||||
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
if rows is not None and hidden_states.shape[0] > 0:
|
||||
hidden_states = _redistribute_input_to_moe_cp(
|
||||
hidden_states, rows, get_moe_cp_size()
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
scattered_states += residual
|
||||
residual = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return read.read(residual, layernorm)
|
||||
@@ -53,6 +53,8 @@ class StageRead(Protocol):
|
||||
# and the residual is left as it is: what a fused add + norm kernel computes,
|
||||
# and what may run on the rows a sum completes onto.
|
||||
norms_plainly: bool
|
||||
# Preserve the read on the source rows before a DP gather.
|
||||
before_gather: bool
|
||||
|
||||
def enter(self, hidden_states) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
@@ -59,6 +59,8 @@ def fold(hidden_states, residual):
|
||||
The following stage receives it with no outstanding residual addition."""
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
if stream.pending is not None and not stream.pending.update.adds_plainly:
|
||||
raise NotImplementedError("fold requires a plain residual update")
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
if residual is not None:
|
||||
@@ -70,10 +72,9 @@ def fold(hidden_states, residual):
|
||||
)
|
||||
|
||||
|
||||
def written(hidden_states, forward_batch):
|
||||
def written(hidden_states):
|
||||
"""Re-enter after a computation that already updated the full residual."""
|
||||
forward_batch.residual_stream = ResidualStream(hidden_states)
|
||||
return hidden_states, forward_batch.residual_stream
|
||||
return hidden_states, ResidualStream(hidden_states)
|
||||
|
||||
|
||||
def finish_layer_stack(
|
||||
@@ -90,16 +91,11 @@ def finish_layer_stack(
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
if isinstance(residual, ResidualStream):
|
||||
stream = forward_batch.residual_stream
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
output = stream.finish(
|
||||
return residual.finish(
|
||||
hidden_states,
|
||||
takes_handoff=final_norm_takes_handoff,
|
||||
preserve_declared=preserve_declared,
|
||||
)
|
||||
forward_batch.residual_stream = None
|
||||
return output
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
@@ -149,7 +145,9 @@ def snapshot(
|
||||
"""Copy a complete output with its plain residual. A statically declared
|
||||
sum is reduced on a copy; the main output and residual remain unchanged."""
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.snapshot(hidden_states, group=group)
|
||||
if group is not None:
|
||||
raise ValueError("a residual stream carries its own reduction")
|
||||
return residual.snapshot(hidden_states)
|
||||
if group is not None:
|
||||
hidden_states = group.all_reduce(hidden_states.clone())
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
"""The plain residual: add and RMSNorm, and the fused all-reduce + add + norm
|
||||
kernels."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
@@ -280,6 +282,14 @@ class Add:
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
class Fp8Input(Enum):
|
||||
"""Additional input forms a projection can consume after a fused norm."""
|
||||
|
||||
TUPLE = auto()
|
||||
TUPLE_AND_BF16 = auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NormQuantRead:
|
||||
"""The input is the residual's norm, fused with the quantization the
|
||||
consumer asks for (``quant_format``) and a post-residual addition; a plain
|
||||
@@ -287,6 +297,8 @@ class NormQuantRead:
|
||||
the norm."""
|
||||
|
||||
norms_plainly = True
|
||||
before_gather: bool = False
|
||||
fp8_input: Optional[Fp8Input] = None
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
@@ -316,11 +328,13 @@ class NormQuantRead:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NormRead:
|
||||
"""The input is the residual's norm; a plain add of the previous output
|
||||
runs in the same kernel. An empty batch skips the norm."""
|
||||
|
||||
norms_plainly = True
|
||||
before_gather: bool = False
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
@@ -19,9 +19,7 @@ from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
|
||||
def start(forward_batch):
|
||||
stream = ResidualStream()
|
||||
forward_batch.residual_stream = stream
|
||||
return stream
|
||||
forward_batch.residual_stream = ResidualStream()
|
||||
|
||||
|
||||
def current(forward_batch):
|
||||
@@ -31,7 +29,8 @@ def current(forward_batch):
|
||||
return stream
|
||||
|
||||
|
||||
def finish(hidden_states, forward_batch):
|
||||
def complete_output(hidden_states, forward_batch):
|
||||
"""Complete the contribution while retaining its pending residual update."""
|
||||
return current(forward_batch).complete(hidden_states)
|
||||
|
||||
|
||||
|
||||
@@ -123,6 +123,7 @@ class _AttentionRead:
|
||||
layer's output: an MHC layer takes its input written back."""
|
||||
|
||||
norms_plainly = False
|
||||
before_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -165,6 +166,7 @@ class _FfnRead:
|
||||
in hc_ffn_post_pre when it takes the batch."""
|
||||
|
||||
norms_plainly = False
|
||||
before_gather = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -158,22 +158,23 @@ class ResidualStream:
|
||||
self.check(hidden)
|
||||
return self.pending.complete() if self.pending is not None else hidden
|
||||
|
||||
def snapshot(self, hidden, *, group=None):
|
||||
def snapshot(self, hidden):
|
||||
self.check(hidden)
|
||||
pending = self.pending
|
||||
if pending is None:
|
||||
return hidden.clone()
|
||||
if not pending.update.adds_plainly:
|
||||
raise NotImplementedError("snapshot requires a plain residual update")
|
||||
if pending.owed is None:
|
||||
value = pending.value
|
||||
elif isinstance(pending.owed, (CarriedSum, DeclaredSum)):
|
||||
value = pending.owed.complete(pending.value.clone())
|
||||
else:
|
||||
raise NotImplementedError("a finalize handoff requires main-output capture")
|
||||
if group is not None:
|
||||
value = group.all_reduce(value.clone())
|
||||
return value.clone() if self.residual is None else value + self.residual
|
||||
|
||||
def finish(self, hidden, *, takes_handoff=False, preserve_declared=False):
|
||||
"""Export the contribution/residual pair; keep the stream available to readers."""
|
||||
self.check(hidden)
|
||||
if self.pending is None:
|
||||
return hidden, None
|
||||
|
||||
@@ -24,12 +24,13 @@ from sglang.srt.layers.communicator.adapters.lora import (
|
||||
publish_attention,
|
||||
publish_ffn,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
from sglang.srt.layers.communicator.contracts import (
|
||||
BatchVariant,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
StageSteps,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import CommunicateSimpleFn
|
||||
from sglang.srt.layers.communicator.ops import gather_attention_tp
|
||||
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
@@ -51,58 +52,40 @@ class StageCommunicator:
|
||||
def __init__(
|
||||
self,
|
||||
plan: "StagePlan",
|
||||
kind: StageKind,
|
||||
norm,
|
||||
*,
|
||||
declaration: Optional["StageDeclaration"] = None,
|
||||
declaration: "StageDeclaration",
|
||||
):
|
||||
self.plan = plan
|
||||
self.kind = kind
|
||||
self.norm = norm
|
||||
self._declaration = declaration
|
||||
self.declaration = declaration
|
||||
|
||||
@property
|
||||
def declaration(self) -> "StageDeclaration":
|
||||
"""Immutable source contract for constructing another local sequence."""
|
||||
if self._declaration is None:
|
||||
raise RuntimeError("this boundary was constructed from raw layout edges")
|
||||
return self._declaration
|
||||
def kind(self):
|
||||
return self.declaration.kind
|
||||
|
||||
@property
|
||||
def norm(self):
|
||||
return self.plan.norm
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
self, forward_batch: ForwardBatch, steps: Optional[StageSteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self.kind.name.lower()} stage"
|
||||
)
|
||||
return entry
|
||||
return steps.entry
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
steps: Optional[StageSteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
stream = (
|
||||
residual
|
||||
if isinstance(residual, ResidualStream)
|
||||
else ResidualStream(residual)
|
||||
)
|
||||
if not isinstance(residual, ResidualStream):
|
||||
if residual is not None or entry.input_sum is not None:
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, call.get("update", ADD), declared_sum=entry.input_sum
|
||||
)
|
||||
elif (
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
@@ -136,8 +119,8 @@ class StageCommunicator:
|
||||
return self.plan.fusions
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
return self.plan.input_rows
|
||||
def incoming_residual_rows(self):
|
||||
return self.plan.incoming_residual_rows
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
@@ -158,11 +141,11 @@ class StageCommunicator:
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
"""Hand this attention's actual contribution to the following stage."""
|
||||
if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output:
|
||||
if self.kind is not StageKind.ATTENTION or not self.plan.direct_handoff:
|
||||
raise RuntimeError(
|
||||
"use the stage exit scope for an FFN or single-stage mixer"
|
||||
)
|
||||
produced = self.plan.produced(self.kind, forward_batch)
|
||||
produced = self.plan.produced(forward_batch)
|
||||
return current(forward_batch).leave(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
@@ -171,24 +154,24 @@ class StageCommunicator:
|
||||
|
||||
def exit(self, forward_batch):
|
||||
result = (
|
||||
self.plan.output.ffn_exit(forward_batch)
|
||||
self.plan.output.ffn_exit(forward_batch, stream=current(forward_batch))
|
||||
if self.kind is StageKind.FFN
|
||||
else self.plan.output.mixer_exit(forward_batch)
|
||||
else self.plan.output.mixer_exit(
|
||||
forward_batch, stream=current(forward_batch)
|
||||
)
|
||||
)
|
||||
result._stream = current(forward_batch)
|
||||
return result
|
||||
|
||||
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
|
||||
hidden_states, residual = from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=self.plan.residual_in_hidden,
|
||||
residual_in_hidden=(
|
||||
self.declaration.previous is not None
|
||||
and self.declaration.previous.update.at_producer
|
||||
),
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
declared_sum = None
|
||||
if forward_batch is not None:
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
entry = steps.attention or steps.ffn
|
||||
declared_sum = entry.input_sum
|
||||
declared_sum = self.entry(forward_batch).input_sum
|
||||
hidden_states, forward_batch.residual_stream = ResidualStream.arrive(
|
||||
hidden_states, residual, ADD, declared_sum=declared_sum
|
||||
)
|
||||
@@ -208,11 +191,9 @@ class StageCommunicator:
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def postprocess(self, hidden_states, forward_batch):
|
||||
hidden_states, stream = self.plan.output.postprocess_layer(
|
||||
return self.plan.output.postprocess_layer(
|
||||
hidden_states, current(forward_batch), forward_batch
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def branch_input(self, source, hidden_states, forward_batch):
|
||||
hidden_states, stream = branch.branch_input(
|
||||
@@ -240,11 +221,14 @@ class StageCommunicator:
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def _prepare_input(self, hidden_states, residual, forward_batch, **call):
|
||||
def _prepare_input(self, hidden_states, stream, forward_batch, **call):
|
||||
plan = self.plan
|
||||
if self.kind is StageKind.ATTENTION:
|
||||
publish_attention(plan._publish_lora_layout)
|
||||
if plan._sp_steps is not None and plan.is_first_layer:
|
||||
if (
|
||||
plan._paths.get(BatchVariant.SEQUENCE_PARALLEL) is not None
|
||||
and plan.enters_stack
|
||||
):
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
@@ -252,7 +236,7 @@ class StageCommunicator:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
else:
|
||||
publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows)
|
||||
return self._prepare(hidden_states, residual, forward_batch, **call)
|
||||
return self._prepare(hidden_states, stream, forward_batch, **call)
|
||||
|
||||
def _prepare_attention(
|
||||
self,
|
||||
@@ -315,11 +299,11 @@ class StageCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
)
|
||||
# Input gathers allocate fresh DP buffers.
|
||||
# Input gathers allocate fresh DP buffers (see gather_attention_tp).
|
||||
# Without a gather this is the mutable residual, so retention copies.
|
||||
captured_last_layer_outputs.capture(
|
||||
gathered_last_layer_output,
|
||||
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
owned=move is gather_attention_tp
|
||||
or (
|
||||
move is None
|
||||
and entry.capture_preserves_residual is not None
|
||||
|
||||
@@ -844,7 +844,7 @@ class BailingMoEModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -57,7 +57,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP, _is_hip
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
@@ -1007,7 +1007,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -181,7 +181,7 @@ class BailingMoEModelNextN(nn.Module):
|
||||
else:
|
||||
hidden_states = self.decoder(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.final_layernorm
|
||||
|
||||
@@ -1395,7 +1395,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
|
||||
@@ -226,7 +226,7 @@ class DeepseekModelNextN(nn.Module):
|
||||
zero_allocator,
|
||||
prev_topk_indices=index_topk_share.topk_indices,
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
|
||||
@@ -2655,7 +2655,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=self._stage_next_sparse,
|
||||
output=output,
|
||||
output_transform=output,
|
||||
),
|
||||
post_attention_layernorm,
|
||||
{"fusions": fusions},
|
||||
@@ -2998,12 +2998,6 @@ class DeepseekV2Model(nn.Module):
|
||||
and layer.mlp.num_fused_shared_experts == 0
|
||||
and not layer.mlp._shared_expert_tp1
|
||||
),
|
||||
# A TP1-replicated output must not ride into the next layer's reduction.
|
||||
requires_local_reduction=lambda layer: (
|
||||
layer.mlp._shared_expert_tp1
|
||||
if isinstance(layer.mlp, DeepseekV2MoE)
|
||||
else layer.mlp.tp_size == 1
|
||||
),
|
||||
label="DeepSeek-V3/GLM",
|
||||
)
|
||||
|
||||
@@ -3160,7 +3154,7 @@ class DeepseekV2Model(nn.Module):
|
||||
proxy_tensors.tensors["topk_indices"] = topk_indices
|
||||
return proxy_tensors
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -1795,7 +1795,7 @@ class Dots3Model(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
|
||||
@@ -119,7 +119,7 @@ class Dot3NoteModelNextN(nn.Module):
|
||||
hidden_states = head.decoder(
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
|
||||
@@ -226,7 +226,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
output=OutputTransform(
|
||||
output_transform=OutputTransform(
|
||||
self.feed_forward.scale_output, before_reduce_scatter=True
|
||||
),
|
||||
sparse=self.is_layer_sparse,
|
||||
@@ -439,7 +439,7 @@ class FalconH1Model(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
|
||||
@@ -583,7 +583,7 @@ class GigaChat35Model(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
@@ -115,7 +115,7 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
|
||||
@@ -1121,7 +1121,7 @@ class Glm4MoeModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -826,7 +826,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -121,7 +121,7 @@ class Glm4MoeLiteModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
|
||||
@@ -105,7 +105,7 @@ class Glm4MoeModelNextN(nn.Module):
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states = self.decoder(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.shared_head.norm
|
||||
|
||||
@@ -801,7 +801,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"residual_in_hidden": residual.ffn_update.at_producer,
|
||||
"qkv_latent_func": self.self_attn.prepare_qkv_latent
|
||||
if not self.is_linear_attn
|
||||
else None,
|
||||
@@ -1192,7 +1191,7 @@ class Glm5NextModel(nn.Module):
|
||||
return PPProxyTensors({"hidden_states": hidden_states})
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -728,7 +728,7 @@ class GptOssModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.srt.layers.communicator import (
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -49,7 +50,6 @@ from sglang.srt.models.qwen3_5 import (
|
||||
Qwen3_5ForCausalLM,
|
||||
Qwen3_5ForConditionalGeneration,
|
||||
Qwen3_5GatedDeltaNet,
|
||||
_enable_qwen35_fused_ar_quant,
|
||||
_linear_accepts_fp8_tuple,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, get_stream
|
||||
@@ -488,18 +488,15 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -623,17 +620,15 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
)
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": False,
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
@@ -793,7 +788,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
input_deepstack_embeds[:, start : start + self.hidden_size],
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
@@ -607,7 +607,7 @@ class LagunaModel(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
|
||||
@@ -761,7 +761,7 @@ class LLaDA2MoeModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -535,7 +535,7 @@ class Llama4Model(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
|
||||
@@ -448,16 +448,16 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
)
|
||||
self.first_ffn, self.second_attn, self.second_ffn = make_stages(
|
||||
(
|
||||
declare_ffn(ordinary_only=True),
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[0],
|
||||
),
|
||||
(
|
||||
declare_attn(ordinary_only=True),
|
||||
declare_attn(),
|
||||
self.input_layernorm[1],
|
||||
{"qkv_latent_func": self.self_attn[1].prepare_qkv_latent},
|
||||
),
|
||||
(
|
||||
declare_ffn(ordinary_only=True),
|
||||
declare_ffn(),
|
||||
self.post_attention_layernorm[1],
|
||||
),
|
||||
prepared_from=self.moe_stage.declaration,
|
||||
|
||||
@@ -272,7 +272,7 @@ class LongcatFlashModelNextN(nn.Module):
|
||||
positions, hidden_states, forward_batch, zero_allocator
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
|
||||
@@ -1069,7 +1069,7 @@ class MiMoV2Model(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
# A draft targeting the final layer ("after layer
|
||||
# num_hidden_layers-1") maps to capture index num_hidden_layers,
|
||||
|
||||
@@ -228,7 +228,7 @@ class MiMoV2ModelNextN(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if forward_batch.return_hidden_states_before_norm:
|
||||
|
||||
@@ -1176,7 +1176,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
@@ -1517,7 +1517,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
|
||||
@@ -753,7 +753,7 @@ class NemotronHModel(nn.Module):
|
||||
return residual_batch.to_pp(
|
||||
hidden_states, forward_batch, preserve_declared=True
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
residual_batch.snapshot(hidden_states, forward_batch)
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
"""Layer-communication helpers for the Nemotron-H model."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE
|
||||
from sglang.srt.layers.communicator import (
|
||||
StageKind,
|
||||
HandoffRows,
|
||||
ProducerReduction,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import NormQuantRead, NormRead
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
ATTN_LAYERS = (MAMBA, ATTENTION)
|
||||
|
||||
@@ -18,33 +19,23 @@ def is_attn_layer(layer_type: str) -> bool:
|
||||
return layer_type in ATTN_LAYERS
|
||||
|
||||
|
||||
def _stage_kind(pattern: str, layer_idx: int) -> Optional[StageKind]:
|
||||
"""Which stage of a decoder layer the stage at ``layer_idx`` stands for: a
|
||||
Mamba or attention mixer the attention, an MLP or MoE the FFN; None past
|
||||
either end."""
|
||||
if not 0 <= layer_idx < len(pattern):
|
||||
return None
|
||||
return StageKind.ATTENTION if is_attn_layer(pattern[layer_idx]) else StageKind.FFN
|
||||
|
||||
|
||||
def _declaration(pattern: str, layer_idx: int):
|
||||
following = _stage_kind(pattern, layer_idx + 1)
|
||||
if is_attn_layer(pattern[layer_idx]):
|
||||
return declare_attn(
|
||||
mixer_exit=True,
|
||||
next_kind=following,
|
||||
ordinary_only=True,
|
||||
read=NormQuantRead(before_gather=True),
|
||||
reduction=ProducerReduction.SCOPED,
|
||||
gathers_tp_input=False,
|
||||
)
|
||||
return declare_ffn(
|
||||
sparse=pattern[layer_idx] == MOE,
|
||||
ordinary_only=True,
|
||||
return_to_attention=True,
|
||||
read=NormRead(before_gather=True),
|
||||
dense_tp_size=get_parallel().tp_size if pattern[layer_idx] != MOE else None,
|
||||
handoff_rows=HandoffRows.ATTENTION,
|
||||
)
|
||||
|
||||
|
||||
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled():
|
||||
raise NotImplementedError("a Nemotron stage with attention CP or LayerNorm SP")
|
||||
@@ -53,9 +44,11 @@ def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
(
|
||||
_declaration(pattern, layer_idx),
|
||||
layer_norm,
|
||||
{"force_layernorm_before_dp_gather": True},
|
||||
),
|
||||
previous=previous,
|
||||
following=_declaration(pattern, layer_idx + 1)
|
||||
if layer_idx + 1 < len(pattern)
|
||||
else None,
|
||||
terminal=layer_idx == len(pattern) - 1,
|
||||
)
|
||||
return boundary
|
||||
|
||||
@@ -1214,7 +1214,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
@@ -1310,14 +1310,13 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states = residual_batch.complete_output(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
|
||||
@@ -477,7 +477,7 @@ class Qwen3Model(Qwen2Model):
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return (
|
||||
|
||||
@@ -29,6 +29,8 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
qwen3_5_gdn_prefill_projection_views,
|
||||
)
|
||||
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
|
||||
|
||||
# Configs
|
||||
from sglang.srt.configs.qwen3_5 import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
@@ -52,12 +54,13 @@ from sglang.srt.layers.communicator import (
|
||||
|
||||
# Layers - Attention
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
|
||||
# Layers - Others
|
||||
from sglang.srt.layers.communicator.residual.add_norm import Fp8Input, NormQuantRead
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
|
||||
# Layers - Others
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
|
||||
# Layers - Linear
|
||||
@@ -259,32 +262,6 @@ if _is_npu:
|
||||
from sgl_kernel_npu.fla.utils import fused_qkvzba_split_reshape_cat_contiguous
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _enable_qwen35_fused_ar_quant() -> bool:
|
||||
"""Gate the fused AR+RMSNorm+per-group-FP8-quant path for Qwen3.5.
|
||||
|
||||
The single-kernel backend is ROCm/aiter/gfx95-only. The model gate stays
|
||||
tied to ROCm/aiter so non-gfx95 HIP can keep the existing 2-kernel fallback
|
||||
behavior for tuple handoff when this branch is used. It replaces the
|
||||
existing ``--enable-aiter-allreduce-fusion`` 3-kernel path
|
||||
(AR → RMSNorm → per-group quant) with either a single fused kernel (when
|
||||
the fully-fused variant is eligible) or a 2-kernel path
|
||||
(fused AR+RMSNorm + separate per-group quant) that still saves one
|
||||
kernel launch vs. baseline. The stage boundary gracefully falls back
|
||||
to ``forward_with_allreduce_fusion`` (plain AR+RMSNorm) when the fused
|
||||
quant helper returns ``None``, so turning this on never regresses the
|
||||
AR+RMSNorm fusion itself.
|
||||
|
||||
Opt-out: set ``SGLANG_DISABLE_FUSED_AR_QUANT=1`` to fall back to the
|
||||
unmodified AR+RMSNorm fusion path.
|
||||
"""
|
||||
if not _use_aiter:
|
||||
return False
|
||||
if get_bool_env_var("SGLANG_DISABLE_FUSED_AR_QUANT", default="false"):
|
||||
return False
|
||||
return bool(get_exec().comm.enable_aiter_allreduce_fusion)
|
||||
|
||||
|
||||
def _linear_accepts_fp8_tuple(linear: nn.Module) -> bool:
|
||||
quant_method = getattr(linear, "quant_method", None)
|
||||
return quant_method.__class__.__name__ == "Fp8LinearMethod" and (
|
||||
@@ -1115,18 +1092,17 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
# GDN layers need both bf16 (for the small in_proj_ba gating
|
||||
# projection) and a quantized tuple only when in_proj_qkvz can consume
|
||||
# it. Otherwise, stay on the plain AR+RMSNorm path.
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant()
|
||||
and _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.linear_attn.in_proj_qkvz)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
fp8_input=Fp8Input.TUPLE_AND_BF16 if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": enable_fused_ar_quant,
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
@@ -1322,17 +1298,17 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
|
||||
# Standard attention layers benefit from a fused quant epilogue only
|
||||
# when qkv_proj can consume the returned quantized tuple.
|
||||
enable_fused_ar_quant = (
|
||||
_enable_qwen35_fused_ar_quant() and _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
)
|
||||
accepts_fp8_input = _linear_accepts_fp8_tuple(self.qkv_proj)
|
||||
boundary_fusions = _layer_fusions(config, is_nextn)
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
declare_attn(
|
||||
read=NormQuantRead(
|
||||
fp8_input=Fp8Input.TUPLE if accepts_fp8_input else None
|
||||
)
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"enable_fused_ar_quant": enable_fused_ar_quant,
|
||||
"fused_ar_quant_keep_bf16": False,
|
||||
"fusions": boundary_fusions,
|
||||
},
|
||||
),
|
||||
@@ -1752,6 +1728,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
# Every layer was checked above.
|
||||
can_defer_finalize=lambda layer: True,
|
||||
label="Qwen3.5",
|
||||
# The final GemmaRMSNorm consumes a finalize handoff.
|
||||
terminal_finalize=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -1099,14 +1099,13 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states = residual_batch.complete_output(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
# norm
|
||||
hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
|
||||
@@ -943,7 +943,7 @@ class Qwen3NextModel(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
|
||||
@@ -1267,7 +1267,7 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
|
||||
@@ -157,7 +157,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
|
||||
@@ -1169,7 +1169,7 @@ class SarvamMLAModel(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
@@ -1272,7 +1272,7 @@ class SarvamMLAForCausalLM(nn.Module):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states = residual_batch.complete_output(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
@@ -1451,7 +1451,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states = residual_batch.complete_output(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
|
||||
@@ -412,7 +412,7 @@ class SDARModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -478,7 +478,7 @@ class SDARMoeModel(nn.Module):
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
@@ -478,7 +478,7 @@ class Step3TextModel(nn.Module):
|
||||
layer = self.layers[i]
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
@@ -711,7 +711,7 @@ class Step3p5Model(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
|
||||
@@ -116,7 +116,7 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
hidden_states_before_norm = None
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# if forward_batch.return_hidden_states_before_norm:
|
||||
|
||||
@@ -1646,7 +1646,7 @@ class XllmModel(nn.Module):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import identity_output
|
||||
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
|
||||
@@ -40,20 +41,14 @@ def make_test_stages(
|
||||
next_sparse=next_sparse,
|
||||
read=residual.ffn_read,
|
||||
update=residual.ffn_update,
|
||||
output=output,
|
||||
output_transform=output,
|
||||
)
|
||||
common = {
|
||||
key: options.pop(key)
|
||||
for key in ("force_layernorm_before_dp_gather", "fusions")
|
||||
if key in options
|
||||
}
|
||||
common = {key: options.pop(key) for key in ("fusions",) if key in options}
|
||||
attn, ffn = make_stages(
|
||||
(
|
||||
attention,
|
||||
attention_norm,
|
||||
dict(
|
||||
residual_in_hidden=residual.ffn_update.at_producer, **common, **options
|
||||
),
|
||||
dict(**common, **options),
|
||||
),
|
||||
(ffn, ffn_norm, common),
|
||||
previous=previous,
|
||||
@@ -69,17 +64,8 @@ def stub_plan():
|
||||
plan.fusions = None
|
||||
plan._publish_lora_layout = False
|
||||
plan._fusion_rows = None
|
||||
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
|
||||
plan.is_first_layer = False
|
||||
plan.is_last_layer = False
|
||||
plan.residual_in_hidden = False
|
||||
# Keep production variant selection while tests replace individual paths.
|
||||
plan._batch_steps = lambda batch: {
|
||||
BatchVariant.ORDINARY: plan._steps,
|
||||
BatchVariant.CONTEXT_PARALLEL: plan._cp_steps,
|
||||
BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps,
|
||||
BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps,
|
||||
}[plan._variant(batch)]
|
||||
plan._paths = {}
|
||||
plan.enters_stack = False
|
||||
return plan
|
||||
|
||||
|
||||
@@ -87,7 +73,10 @@ def stub_stage(plan, kind):
|
||||
"""Keep the same concrete boundary when a test patches its methods."""
|
||||
stages = plan.__dict__.setdefault("_stub_stages", {})
|
||||
if kind not in stages:
|
||||
stages[kind] = StageCommunicator(plan, kind, plan.norm)
|
||||
stages[kind] = StageCommunicator(
|
||||
plan,
|
||||
declaration=(declare_attn() if kind.name == "ATTENTION" else declare_ffn()),
|
||||
)
|
||||
return stages[kind]
|
||||
|
||||
|
||||
@@ -95,18 +84,16 @@ def sp_region_steps():
|
||||
"""Local SP rows with no owed sum, for tests of activation and exits."""
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
BoundarySteps,
|
||||
CommunicateSummableTensorPairFn,
|
||||
EdgeDecl,
|
||||
Layout,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
StageSteps,
|
||||
TokenAxis,
|
||||
make_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input
|
||||
from sglang.srt.layers.communicator.prepare import _hand_qkv_hook_its_input
|
||||
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
output = StageOutput(rows)
|
||||
@@ -117,10 +104,62 @@ def sp_region_steps():
|
||||
)
|
||||
return StageEntry(selected.prepare, rows, handoff=handoff)
|
||||
|
||||
return BoundarySteps(
|
||||
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input),
|
||||
entry(NORM_READ),
|
||||
output,
|
||||
CommunicateSummableTensorPairFn._trivial,
|
||||
False,
|
||||
return StageSteps(
|
||||
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input), output, identity_output
|
||||
)
|
||||
|
||||
|
||||
def prepare_input(stage, hidden, residual, forward_batch, **call):
|
||||
return prepare_raw(stage, "_prepare_input", hidden, residual, forward_batch, **call)
|
||||
|
||||
|
||||
def prepare_attention(stage, hidden, residual, forward_batch, *args, **call):
|
||||
return prepare_raw(
|
||||
stage, "_prepare_attention", hidden, residual, forward_batch, *args, **call
|
||||
)
|
||||
|
||||
|
||||
def prepare_raw(stage, method, hidden, residual, forward_batch, *args, **call):
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
|
||||
if not isinstance(residual, ResidualStream):
|
||||
stream = ResidualStream(residual)
|
||||
if residual is not None:
|
||||
hidden = stream.leave(
|
||||
hidden,
|
||||
call.get("update", ADD),
|
||||
declared_sum=stage.entry(forward_batch).input_sum,
|
||||
)
|
||||
else:
|
||||
stream = residual
|
||||
return getattr(stage, method)(hidden, stream, forward_batch, *args, **call)
|
||||
|
||||
|
||||
def finish_exit(scope, hidden, residual):
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
|
||||
if isinstance(residual, ResidualStream):
|
||||
scope._stream = residual
|
||||
return scope.finish(hidden), residual
|
||||
scope._stream.residual = residual
|
||||
output = scope.finish(hidden)
|
||||
if scope._stream.pending is None:
|
||||
return output, None
|
||||
return scope._stream.input(output)
|
||||
|
||||
|
||||
def postprocess_output(boundary, hidden, residual, forward_batch):
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
|
||||
stream = (
|
||||
residual if isinstance(residual, ResidualStream) else ResidualStream(residual)
|
||||
)
|
||||
hidden = boundary.postprocess_layer(hidden, stream, forward_batch)
|
||||
if isinstance(residual, ResidualStream):
|
||||
return hidden, stream
|
||||
return (hidden, None) if stream.pending is None else stream.input(hidden)
|
||||
|
||||
|
||||
def identity_input(hidden_states, forward_batch):
|
||||
return hidden_states
|
||||
|
||||
@@ -12,19 +12,19 @@ from sglang.srt.layers import communicator as communicator
|
||||
from sglang.srt.layers.communicator import (
|
||||
boundary,
|
||||
construction,
|
||||
contracts,
|
||||
exit,
|
||||
factories,
|
||||
layer,
|
||||
layout,
|
||||
legacy_stage,
|
||||
ops,
|
||||
output,
|
||||
prepare,
|
||||
)
|
||||
from sglang.srt.layers.communicator import residual as residual_contract
|
||||
from sglang.srt.layers.communicator import (
|
||||
stage,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters import attention, branch, lora
|
||||
from sglang.srt.layers.communicator.adapters import attention, branch, lora, overlap
|
||||
from sglang.srt.layers.communicator.fusions import allreduce
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
access,
|
||||
@@ -40,6 +40,7 @@ COMMUNICATOR_MODULES = (
|
||||
exit,
|
||||
branch,
|
||||
lora,
|
||||
overlap,
|
||||
layout,
|
||||
output,
|
||||
attention,
|
||||
@@ -50,10 +51,10 @@ COMMUNICATOR_MODULES = (
|
||||
ops,
|
||||
boundary,
|
||||
construction,
|
||||
contracts,
|
||||
prepare,
|
||||
factories,
|
||||
stage,
|
||||
layer,
|
||||
legacy_stage,
|
||||
batch,
|
||||
stream,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user