[Refactor] Construct independent decoder stage boundaries (#41552)

This commit is contained in:
Cheng Wan
2026-09-28 20:17:00 -07:00
committed by GitHub
parent acffb0d20d
commit 49b5e76e45
79 changed files with 5182 additions and 2318 deletions
@@ -0,0 +1,88 @@
"""Resolve collective preferences before constructing target and draft models."""
import json
from sglang.srt.arg_groups.model_override_base import model_config_of
_RSV_ARCHITECTURES = {
"BailingMoELinearForCausalLM",
"BailingMoeV2_5ForCausalLM",
"LongcatFlashForCausalLM",
"LongcatFlashForCausalLMNextN",
"Step3VLForConditionalGeneration",
}
_AR_ARCHITECTURES = {
"Qwen3ForCausalLM",
"Qwen3ForSequenceClassification",
"Qwen3Model",
"Qwen3ForRewardModel",
"MossVLForCausalLM",
"MossVLForConditionalGeneration",
}
def model_boundary_reduction(config):
"""One default for a model's dense and MoE layers, including its MTP."""
architectures = set(getattr(config, "architectures", None) or ())
if architectures & _RSV_ARCHITECTURES:
return "rsv"
if architectures & _AR_ARCHITECTURES:
return "ar"
# MiniCPM-V 4.5 constructs Qwen3 from its top-level config.
if (
getattr(config, "model_type", None) == "minicpmv"
and str(getattr(config, "version", "")) == "4.5"
):
return "ar"
# Wrappers carry their actual language backbone in the nested HF config.
# Match Qwen3 specifically: Qwen3-MoE/Next/3.5 fall through to rs+rsv.
if getattr(config, "model_type", None) in ("qwen3", "qwen3_vl_text"):
return "ar"
get_text = getattr(config, "get_text_config", None)
if get_text is not None:
text = get_text()
if text is not config:
return model_boundary_reduction(text)
for name in ("text_config", "llm_config", "language_config"):
text = getattr(config, name, None)
if text is not None and text is not config:
return model_boundary_reduction(text)
return "rs+rsv"
def resolve_boundary_reduction(view):
"""Resolve auto once, after speculative paths/revisions have been resolved."""
requested = view.boundary_reduction
dummy = view.model_path.lower() in ("none", "dummy")
if requested != "auto":
target = draft = requested
else:
target = draft = (
"rs+rsv"
if dummy
else model_boundary_reduction(model_config_of(view).hf_config)
)
draft_path = view.speculative_draft_model_path
if (
not dummy
and view.speculative_algorithm is not None
and draft_path is not None
and draft_path != view.model_path
):
from sglang.srt.utils.hf_transformers_utils import get_config
kwargs = {}
if view.decrypted_draft_config_file:
kwargs["_configuration_file"] = view.decrypted_draft_config_file
config = get_config(
draft_path,
trust_remote_code=view.trust_remote_code,
revision=view.speculative_draft_model_revision,
model_override_args=json.loads(view.json_model_override_args),
**kwargs,
)
draft = model_boundary_reduction(config)
return {
"boundary_reduction": target,
"speculative_boundary_reduction": draft,
}
@@ -250,6 +250,7 @@ POSITIONAL_FIELD_ORDER = (
"enable_torch_symm_mem",
"enable_scattered_sconv",
"pre_warm_nccl",
"boundary_reduction",
"enable_quant_communications",
"enable_flashinfer_allreduce_fusion",
"enforce_disable_flashinfer_allreduce_fusion",
@@ -620,6 +620,19 @@ class ExecComm(msgspec.Struct):
bool,
"Pre-warm NCCL/RCCL communicators during startup to reduce P99 TTFT cold-start latency. Default: enabled for AMD/HIP (RCCL), disabled for NVIDIA/CUDA (NCCL).",
] = False
boundary_reduction: A[
Literal["auto", "ar", "rs", "rsv", "rs+rsv"],
Arg(
help="Select FFN boundary reduction: ar uses all-reduce then token "
"redistribution; rs and rsv permit fixed-size and variable-size "
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
"Unsupported paths fall back to ar. The option applies only to FFN "
"layers using stage boundaries, where auto resolves the model default. "
"Required attention and MoE collectives and "
"all-reduce fusion are unaffected.",
resolvable=True,
),
] = "auto"
enable_quant_communications: A[
Optional[bool],
"Enable INT8 quantization of TP communications (limited support).",
@@ -39,6 +39,10 @@ class Spec(msgspec.Struct):
aliases=["--speculative-draft-model"],
),
] = None
speculative_boundary_reduction: A[
Literal["ar", "rs", "rsv", "rs+rsv"],
Arg(no_cli=True, resolvable=True),
] = "rs+rsv"
speculative_draft_model_revision: A[
Optional[str],
"The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.",
+7
View File
@@ -92,6 +92,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(validate_prefill_cp_platform, server_args)
run_hook(handle_hardware_runtime_validation, server_args)
if cfg.model_path.lower() in ["none", "dummy"]:
from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
run_post_process_pass(server_args, resolve_boundary_reduction)
return
from sglang.srt.arg_groups.model_path_hook import (
@@ -284,6 +287,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_speculative_decoding, server_args)
from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
run_post_process_pass(server_args, resolve_boundary_reduction)
# After the speculative hook so speculative_algorithm is final.
from sglang.srt.arg_groups.layernorm_sp_hook import handle_layernorm_sp
@@ -15,7 +15,6 @@ from sglang.srt.batch_overlap.operations import (
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.communicator import (
CommunicateContext,
Layout,
tbo_split_moves,
)
@@ -1022,14 +1021,12 @@ def _model_forward_tbo_split_inputs(
zero_allocator: Optional[BumpAllocator],
layer_input_rows: Layout,
) -> List[Dict]:
context = CommunicateContext.init_new()
to_splitter, to_layer_input = tbo_split_moves(layer_input_rows)
hidden_states, residual = to_splitter(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=context,
)
inputs_arr = _model_forward_tbo_split_inputs_raw(
@@ -1045,7 +1042,6 @@ def _model_forward_tbo_split_inputs(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=context,
)
return dict(
hidden_states=hidden_states,
@@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Communication between a decoder layer's stages."""
"""Stage boundaries: layout transport, residual updates, reads, and fusion."""
from sglang.srt.layers.communicator.adapters.attention import (
AttentionInputs,
@@ -19,7 +19,6 @@ from sglang.srt.layers.communicator.adapters.attention import (
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
DecoderLayerSides,
EdgeDecl,
FusedMlpInput,
LayerStage,
@@ -28,25 +27,25 @@ from sglang.srt.layers.communicator.boundary import (
StageInput,
StageKind,
StageOutput,
decoder_layer_edges,
decoder_layer_sides,
input_scattered_layer_sides,
make_boundary,
make_output_boundary,
sequence_parallel_layer_sides,
stage_edges,
tbo_split_moves,
)
from sglang.srt.layers.communicator.layer import (
FfnCompletion,
FfnExit,
LayerCommunicator,
LayerFacts,
MHCLayerCommunicator,
MixerExit,
from sglang.srt.layers.communicator.construction import (
BatchVariant,
StageEdges,
make_stage,
)
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
from sglang.srt.layers.communicator.factories import (
declare_attn,
declare_ffn,
make_attn_stage,
make_ffn_stage,
make_stages,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -79,25 +78,28 @@ from sglang.srt.layers.communicator.residual.mhc import (
)
__all__ = [
"LayerCommunicator",
"MHCLayerCommunicator",
"LayerFacts",
"declare_attn",
"declare_ffn",
"make_attn_stage",
"make_ffn_stage",
"make_stages",
"ADD",
"AttentionInputs",
"BoundarySteps",
"CommunicateContext",
"CommunicateSimpleFn",
"CommunicateSummableTensorPairFn",
"DecoderLayerSides",
"EdgeDecl",
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
"FfnCompletion",
"FfnExit",
"FusedMlpInput",
"HandoffOutput",
"LayerCommunicator",
"LayerResidual",
"LayerFacts",
"LayerStage",
"Layout",
"MHCLayerCommunicator",
"MHCState",
"MixerExit",
"NORM_QUANT_READ",
@@ -111,20 +113,22 @@ __all__ = [
"SumGroup",
"TokenAxis",
"UnreducedOutput",
"decoder_layer_edges",
"decoder_layer_sides",
"enable_moe_dense_fully_dp",
"get_attn_tp_context",
"input_scattered_layer_sides",
"make_boundary",
"make_output_boundary",
"moe_cp_gathers_sparse_moe_input",
"move_rows",
"reduce_output",
"sequence_parallel_layer_sides",
"sparse_moe_gathers_over_moe_cp",
"stage_edges",
"tbo_split_moves",
"token_axis_sizes",
"tp_reduce_scatter",
]
from sglang.srt.layers.communicator.layer import (
LayerCommunicator,
LayerFacts,
MHCLayerCommunicator,
)
@@ -21,7 +21,6 @@ from typing import Callable
import torch
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -182,7 +181,6 @@ def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
return gathered
def _redistribute_to_attn_tp_shards(
tensor: torch.Tensor, context: CommunicateContext
) -> torch.Tensor:
return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
def _redistribute_to_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
parallel = get_parallel()
return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank]
@@ -0,0 +1,87 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Move prepared branch inputs and merge complete branch contributions."""
from __future__ import annotations
from typing import Tuple
import torch
from sglang.srt.layers.communicator.ops import move_rows
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
def branch_input(
self,
source: object,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""The FFN input and residual that ``source``'s boundary read for its
own FFN, for this layer's FFN, which branches from the same input: moved
to the rows this FFN needs and its residual's rows."""
rows, residual_rows, _ = source._branch_rows(forward_batch)
to, residual_to, _ = self._branch_rows(forward_batch)
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
if stream.pending is not None:
raise RuntimeError("a branch must start from a prepared stage input")
residual = stream.residual
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
return (
hidden_states,
ResidualStream(residual) if stream is not None else residual,
)
def branch_output(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
"""This layer's complete FFN output as a branch's contribution, which
adds to the layer's output without writing the residual: moved to the
rows the layer hands on."""
rows, _, to = self._branch_rows(forward_batch)
return move_rows(hidden_states, rows, to, forward_batch)
def merge_branch(
self,
contribution: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor,
source: object,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""A contribution from ``branch_output`` summed with what ``source``'s
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
this layer hands on."""
_, _, rows = source._branch_rows(forward_batch)
_, _, to = self._branch_rows(forward_batch)
stream = residual if isinstance(residual, ResidualStream) else None
update = None
if stream is not None:
stream.check(hidden_states)
update = stream.pending.update
hidden_states, residual = stream.finish(hidden_states)
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
residual = move_rows(residual, rows, to, forward_batch)
output = contribution + hidden_states
if stream is not None:
stream.write(residual)
return stream.leave(output, update), stream
return output, residual
@@ -0,0 +1,32 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Publish the token rows consumed by the selected stage to LoRA kernels."""
from sglang.srt.layers.communicator.layout import TokenAxis
from sglang.srt.runtime_context import LoRABatchLayout, get_forward
def publish_attention(enabled):
if enabled:
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
def publish_ffn(enabled, input_rows):
if enabled:
get_forward().set(
"lora_batch_layout",
LoRABatchLayout.DP_LOCAL
if TokenAxis.ATTN_DP in input_rows.sharded
else LoRABatchLayout.TP_GLOBAL,
)
@@ -51,6 +51,7 @@ from sglang.srt.layers.communicator.ops import (
tp_reduce_scatter,
tp_slice,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import (
LayerResidual,
StageRead,
@@ -102,6 +103,7 @@ class StageOutput(msgspec.Struct, frozen=True):
# How the producer's output is written into the residual. An arrival
# description has no producer object; its edge declares capabilities only.
update: Optional[StageUpdate] = ADD
transform: Optional[OutputTransform] = None
class StageDecl(msgspec.Struct, frozen=True):
@@ -481,6 +483,7 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
# The group whose sum it completes.
completes: SumGroup
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
preserves_residual: Optional[Callable] = None
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
@@ -536,12 +539,12 @@ class StageEntry(msgspec.Struct, frozen=True):
# Completes what the input owes, writes the previous stage's output into
# the residual and reads this stage's input:
# (hidden_states, residual, forward_batch, norm, context, **call).
# (hidden_states, residual, forward_batch, norm, **call).
prepare: Callable
# The rows prepare hands the stage.
input_rows: Layout
# Moves the input onto the stage's rows after prepare, when prepare does
# not: (hidden_states, forward_batch, context) -> hidden_states.
# not: (hidden_states, forward_batch) -> hidden_states.
input_move: Optional[Callable] = None
# Hands the stage its input once it is on its rows:
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
@@ -551,6 +554,7 @@ class StageEntry(msgspec.Struct, frozen=True):
# Return the updated residual to the producer's rows for aux capture.
capture_move: Optional[Callable] = None
capture_move_allocates: bool = False
preserves_residual: Optional[Callable] = None
capture_preserves_residual: Optional[Callable] = None
input_sum: Optional[SumGroup] = None
@@ -619,6 +623,7 @@ class Boundary(msgspec.Struct, frozen=True):
output_move: Optional[Callable] = None
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
preserves_residual: Optional[Callable] = None
@property
def input_rows(self) -> Layout:
@@ -682,6 +687,7 @@ def make_boundary(
raise NotImplementedError("update capabilities require different input moves")
return msgspec.structs.replace(
first,
preserves_residual=None,
prepare=partial(
_dispatch_consumer,
paths={capability: path.prepare for capability, path in paths.items()},
@@ -756,8 +762,24 @@ def _bind_consumer(
cp_moves=cp_moves,
enters_stack=False,
)
preserves_residual = None
if (
edge.produced.always_leaves
and edge.produced.group is SumGroup.ATTN_TP
and isinstance(step, partial)
and step.func is _mlp_input_without_dp
and not step.keywords["gathers_residual"]
):
selected = step.keywords["fusions"]
if selected:
# Only the first reachable candidate can certify ownership; an
# earlier custom candidate could otherwise mutate the residual.
preserves_residual = next(
f.preserves_residual for f in fusions if f.run == selected[0]
)
return Boundary(
edge,
preserves_residual=preserves_residual,
prepare=partial(
_consumer_step,
step=step,
@@ -0,0 +1,307 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Bind one stage to predeclared boundaries and reusable batch paths."""
from dataclasses import dataclass
from enum import Enum, auto
from functools import cached_property
from typing import Callable, Optional, Protocol, Tuple
from sglang.srt.environ import envs
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
EdgeDecl,
FusedMlpInput,
StageDecl,
StageEntry,
StageKind,
make_boundary,
make_output_boundary,
)
from sglang.srt.layers.communicator.exit import OutputBoundary
from sglang.srt.layers.communicator.fusions.allreduce import (
attention_fusions,
ffn_fusions,
)
from sglang.srt.layers.communicator.layout import (
TokenAxis,
_batch_shards_over_cp,
_gathers_over_attention_cp,
_generic_prefill_cp_shards_tokens,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
get_forward,
get_lora,
get_parallel,
get_spec,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
class BoundaryFusions(Protocol):
"""The fused kernels a fusion backend gives a layer, bound to it and tried
before the layer's own: at its attention input (each takes what the previous
layer left, the residual, the batch and a post-residual addition), at its
FFN input, and at its FFN exit (each says what the exit does when its kernel
takes the batch)."""
def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ...
def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ...
requires_local_reduction: bool
def can_defer_finalize(
self, layer: "StagePlan", forward_batch: ForwardBatch
) -> bool: ...
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
"""A MoE layer under attention CP whose tokens the steps cannot bring
to it: under a prefill CP, one dispatched per DP shard under attention DP
and GQA CP, and one on the TP group whose data-parallel groups are the CP
ranks under DSA or MLA CP; and under attention DP, one on the TP group
whose data-parallel groups are the CP ranks."""
parallel = get_parallel()
gqa = not _gathers_over_attention_cp()
if moe_on_local_rows:
if cp_shards and gqa and parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
)
elif parallel.moe_dp_size == parallel.attn_cp_size:
if cp_shards and not gqa:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"DSA or MLA prefill CP"
)
if parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"attention DP and attention CP"
)
def _input_can_be_scattered() -> bool:
"""Whether a batch may run this layer with input-scattered attention:
configured, on TP without attention DP, a prefill CP, an a2a backend or
a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
known once the model is built."""
parallel = get_parallel()
return (
parallel.enable_attn_tp_input_scattered
and parallel.tp_size > 1
and parallel.attn_dp_size == 1
and not _generic_prefill_cp_shards_tokens()
and get_moe_a2a_backend().is_none()
and not enable_moe_dense_fully_dp()
)
class BatchVariant(Enum):
ORDINARY = auto()
CONTEXT_PARALLEL = auto()
INPUT_SCATTERED = auto()
SEQUENCE_PARALLEL = auto()
@dataclass(frozen=True)
class StageEdges:
"""One stage's incoming and outgoing declarations for one batch variant.
No adjacent stage object is required. Edges can come from a local model
assembler, pipeline contract or branch adapter.
"""
incoming: EdgeDecl
outgoing: EdgeDecl
handoff: Optional[Callable] = None
cp_moves: object = None
def _requires_branch_input(*args, **kwargs):
raise RuntimeError("a prepared branch must enter through branch_input, not prepare")
class StagePlan:
"""Precomputed paths for one stage; never owns the neighbouring norm."""
def __init__(
self,
kind,
norm,
variants,
*,
enters_stack=False,
prepared_input=False,
terminal=False,
fixed_output=False,
is_sparse=False,
qkv_latent_func=None,
force_layernorm_before_dp_gather=False,
enable_fused_ar_quant=False,
fused_ar_quant_keep_bf16=False,
residual_in_hidden=False,
fusions=None,
):
self.kind = kind
self.norm = norm
self.variants = dict(variants)
self.is_first_layer = enters_stack
self.is_last_layer = terminal
self.fixed_output = fixed_output
self.is_sparse = is_sparse
self.qkv_latent_func = qkv_latent_func
self.enable_fused_ar_quant = enable_fused_ar_quant
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
self.residual_in_hidden = residual_in_hidden
self.fusions = fusions
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
get_lora().enable_lora
)
self._fusion_rows = None
carried = attention_fusions(self) if kind is StageKind.ATTENTION else ()
self._carried_fusions = carried
fused = ffn_fusions(self) if kind is StageKind.FFN else ()
self._paths = {}
for variant, edges in self.variants.items():
if prepared_input:
# The branch adapter moves an already-read input and forks its
# stream. Binding a normal prepare would add or norm it twice.
entry = StageEntry(
prepare=_requires_branch_input,
input_rows=edges.incoming.need.layout,
)
else:
into = make_boundary(
edges.incoming,
fusions=fused,
carried_fusions=carried,
force_layernorm_before_gather=force_layernorm_before_dp_gather,
cp_moves=edges.cp_moves,
enters_stack=enters_stack,
)
entry = StageEntry(
prepare=into.prepare,
input_rows=into.input_rows,
input_move=into.input_move,
handoff=edges.handoff,
fused=into.fused,
capture_move=into.capture_move,
capture_move_allocates=into.capture_move_allocates,
preserves_residual=into.preserves_residual,
input_sum=edges.incoming.produced.group
if edges.incoming.produced.always_leaves
else None,
)
out = (
None
if fixed_output
else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves)
)
self._paths[variant] = BoundarySteps(
attention=entry if kind is StageKind.ATTENTION else None,
ffn=entry if kind is StageKind.FFN else None,
ffn_output=edges.outgoing.produced,
ffn_output_move=None if out is None else out.output_move,
ffn_output_move_completes_sum=False
if out is None
else out.output_move_completes_sum,
ffn_sum_is_movable=edges.outgoing.produced.group is not None,
)
self._steps = self._paths[BatchVariant.ORDINARY]
self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL)
self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED)
self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL)
@property
def input_rows(self):
return self.variants[BatchVariant.ORDINARY].incoming.residual
@property
def input_on_attention_tp_slices(self):
return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
def _variant(self, forward_batch):
if self._sp_steps is not None and get_forward().sp_active:
return BatchVariant.SEQUENCE_PARALLEL
if (
self._input_scattered_steps is not None
and get_attn_tp_context().input_scattered
):
return BatchVariant.INPUT_SCATTERED
if self._cp_steps is not None and _batch_shards_over_cp(forward_batch):
return BatchVariant.CONTEXT_PARALLEL
return BatchVariant.ORDINARY
def _batch_steps(self, forward_batch):
return self._paths[self._variant(forward_batch)]
def fusion_rows(self, forward_batch):
if self._fusion_rows is not None:
return self._fusion_rows[self._variant(forward_batch)]
entry = self._batch_steps(forward_batch)
return (entry.attention or entry.ffn).input_rows
def produced(self, kind, forward_batch):
return self._batch_steps(forward_batch).ffn_output
@cached_property
def output(self):
return OutputBoundary(self)
def _branch_rows(self, forward_batch):
if self._variant(forward_batch) is not BatchVariant.ORDINARY:
raise NotImplementedError("branch transport requires ordinary batch rows")
edges = self.variants[BatchVariant.ORDINARY]
return (
edges.incoming.need.layout,
edges.incoming.residual_to,
edges.outgoing.need.layout,
)
def make_stage(
declaration: StageDecl,
*,
kind: StageKind,
norm,
incoming: EdgeDecl,
outgoing: EdgeDecl,
variants=None,
handoff=None,
**options,
) -> StageCommunicator:
"""Build one real stage from its own declaration and adjacent edges.
The optional variants supply the same stage's CP/SP/input-scattered
declarations. Neither a paired stage nor its layer facts/norm is needed.
"""
if incoming.need != declaration.input or outgoing.produced != declaration.output:
raise ValueError("stage declaration disagrees with its boundaries")
paths = dict(variants or {})
paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff)
return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm)
@@ -0,0 +1,521 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Producer output choices, reduction scopes, and completion."""
from __future__ import annotations
from functools import partial
from typing import Callable, Optional, Tuple, Union
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.environ import envs
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
from sglang.srt.layers.communicator.layout import (
SumGroup,
_ffn_has_tokens,
_sum_group,
)
from sglang.srt.layers.communicator.ops import (
_all_reduce_then_to_local_tokens,
_redistribute_output,
_reduce_and_redistribute_output_max_len,
_reduce_and_redistribute_output_varlen,
_to_local_tokens,
)
from sglang.srt.layers.communicator.output import (
UnreducedOutput,
)
from sglang.srt.layers.communicator.residual.add_norm import (
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.layers.dp_attention import (
can_use_dp_reduce_scatter,
is_dp_attention_enabled,
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
post_experts_reduction_group,
post_experts_sum_is_one_all_reduce,
should_use_dp_reduce_scatterv,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_parallel,
)
def _reduce_and_redistribute_output_step(
forward_batch: ForwardBatch,
*,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings an FFN output gathered over attention
DP back to this rank's tokens when the FFN leaves its sum to it (see
StageOutput); None when the FFN reduces the output and only a scatter
remains."""
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
return _reduce_and_redistribute_output_varlen
if (
leaves_for_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
return _reduce_and_redistribute_output_max_len
return None
class OutputBoundary:
"""Choose and finish one producer's output; the batch owns its residual."""
def __init__(self, plan):
self.plan = plan
def postprocess_layer(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
"""Move a complete output from the operation-scheduled producer path."""
stream = residual if isinstance(residual, ResidualStream) else None
hidden_states, residual = self._complete_ffn_output_now(
hidden_states,
stream.residual if stream is not None else residual,
forward_batch=forward_batch,
dp_step=None,
)
return self._leave_ffn_output(
hidden_states,
residual,
stream,
self.plan._batch_steps(forward_batch).ffn_output.update,
)
@staticmethod
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
if stream is None:
return hidden_states, residual
stream.residual = residual
if update.at_producer:
return stream.write(hidden_states), stream
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
@staticmethod
def _declared_ffn_sum(steps, skipped_reduction):
# The input-scattered path leaves the sum for the next input's TP
# reduce-scatter. Other skip paths complete it in the output move.
if (
skipped_reduction
and not steps.returns_over_dp
and not steps.ffn_output_move_completes_sum
):
return SumGroup.TP
return None
def _local_token_move_can_go_to_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, when
the next layer's input also writes the output into the residual."""
steps = self.plan._batch_steps(forward_batch)
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings this layer's FFN output back to this
rank's tokens under attention DP; None when the base postprocess would
only scatter, or does not move tokens."""
steps = self.plan._batch_steps(forward_batch)
if not steps.returns_over_dp:
return None
return _reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
)
def _ffn_leaves_sum_to_reduce_scatter(
self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
) -> bool:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
steps = self.plan._batch_steps(forward_batch)
if dp_step is not None:
return True
if not steps.ffn_output.leaves_for_reduce_scatter:
return False
if steps.ffn_output_move_completes_sum:
return True
return get_attn_tp_context().input_scattered and not self.plan.is_last_layer
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
"""The group this layer's FFN output owes its sum over: the MoE output's
group on a sparse layer, the TP group a dense MLP reduces over."""
return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group)
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
"""Decide once, before the FFN runs, what it skips and what completes its
output: the next layer's input, or this layer's postprocess step."""
dp_step = self._postprocess_dp_step(forward_batch)
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
forward_batch, dp_step
)
complete_now = partial(
self._complete_ffn_output_now,
forward_batch=forward_batch,
dp_step=dp_step,
)
steps = self.plan._batch_steps(forward_batch)
if not steps.ffn_output.leaves_for_next_layer:
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete_now,
)
defer_moe_finalize = (
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
)
# Producers declare remaining work independently of the kernel chosen
# by the consumer. Every handoff also carries an unfused completion.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
)
if fuse_mlp_allreduce:
group = self.ffn_reduction_group(forward_batch)
if steps.returns_over_dp:
# Under attention DP the next layer also brings the sum back to
# this rank's tokens.
wrap = partial(
UnreducedOutput,
reduce_and_redistribute=partial(
_all_reduce_then_to_local_tokens, group, forward_batch
),
)
else:
wrap = partial(UnreducedOutput, group=group)
complete = partial(_leave_to_next_layer, wrap)
elif (
dp_step is not None
and not self.plan.is_last_layer
and self._local_token_move_can_go_to_next_layer(forward_batch)
):
complete = partial(
_leave_to_next_layer,
partial(
UnreducedOutput,
reduce_and_redistribute=partial(
_to_local_tokens, dp_step, forward_batch
),
),
)
else:
complete = complete_now
return FfnCompletion(
defer_moe_finalize=defer_moe_finalize,
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete,
)
def _complete_ffn_output_now(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
*,
forward_batch: ForwardBatch,
dp_step: Optional[Callable],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step already
chosen: the move back to where the next layer reads the FFN output, then
the write-back into the residual for a layer that does it itself."""
steps = self.plan._batch_steps(forward_batch)
if steps.ffn_output.transform is not None:
hidden_states = steps.ffn_output.transform.apply(hidden_states)
if steps.returns_over_dp:
hidden_states = _to_local_tokens(
dp_step or _redistribute_output, forward_batch, hidden_states
)
else:
hidden_states, residual = steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
update = steps.ffn_output.update
if residual is not None and update.at_producer:
hidden_states = update.update(hidden_states, residual)
residual = None
return hidden_states, residual
def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit:
"""Decide once whether this stage's mixer (an attention-like stage)
skips its output all-reduce. Use the result as a context manager around
the mixer, then call ``finish``."""
return MixerExit(self, forward_batch)
def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit:
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
return FfnExit(self, forward_batch)
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if (
(
self.plan.fusions is not None
and self.plan.fusions.requires_local_reduction
)
or is_enable_moe_cp_allgather()
or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable
):
return False
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
# two disjoint groups; post_experts_all_reduce() merges them into one
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
# When merging is blocked, no single group covers both, so fusion stays off.
parallel = get_parallel()
if (
parallel.moe_ep_size > 1
and parallel.moe_tp_size > 1
and not can_merge_post_experts_all_reduce()
):
return False
if (
is_dp_attention_enabled()
and self.plan._speculative_algo is not None
and self.plan._speculative_algo.is_eagle()
):
return False
return not get_attn_tp_context().input_scattered
def _ffn_sum_moves_to_next_layer(
self,
forward_batch: ForwardBatch,
*,
mlp_reduce_scatter: bool,
dp_step: Optional[Callable],
) -> bool:
"""Whether the FFN leaves its output's all-reduce to the next layer's
input when no fused kernel takes it: when the next layer would run the
same all-reduce the FFN itself would have."""
return (
get_parallel().tp_size > 1
and not self.plan.is_last_layer
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and _can_defer_ffn_reduction(forward_batch, self.plan)
and not mlp_reduce_scatter
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
and (
not is_dp_attention_enabled()
or (
self._local_token_move_can_go_to_next_layer(forward_batch)
and dp_step is None
)
)
)
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
A fused-kernel fallback completes the partial-output contract selected
before the producer ran, without recomputing the producer's LoRA path.
"""
if not _ffn_has_tokens(forward_batch):
return False
if post_experts_sum_is_one_all_reduce():
return True
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
# when the sum is deferred. Preserve these fused paths, but keep their
# producer-side reduction order when no fused consumer is enabled.
if not (
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
and get_moe_a2a_backend().is_none()
and not get_exec().comm.enable_quant_communications
and post_experts_reduction_group() is get_parallel().tp_group
):
return False
if (
boundary is not None
and boundary.fusions is not None
and boundary.fusions.can_defer_all_reduce(boundary, forward_batch)
):
return True
if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]):
return True
# Aiter also checks width and bytes, so use the actual residual storage.
residual = forward_batch.residual_stream.residual
return residual is not None and apply_aiter_all_reduce_fusion(
residual, forward_batch
)
class FfnCompletion(msgspec.Struct, frozen=True):
"""One FFN's reduction decision. The flags are published while the FFN runs;
``complete(hidden_states, residual)`` then either wraps the output for the
next layer's input to complete, or runs this layer's postprocess step."""
defer_moe_finalize: bool
fuse_mlp_allreduce: bool
mlp_reduce_scatter: bool
complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
def _leave_to_next_layer(
wrap: Callable[[torch.Tensor], UnreducedOutput],
hidden_states: torch.Tensor,
residual: torch.Tensor,
) -> Tuple[UnreducedOutput, torch.Tensor]:
return wrap(hidden_states), residual
class MixerExit:
"""The scope that publishes a mixer's decision while it runs: inside the
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
row-parallel output projection to skip the all-reduce. It skips when the
stage's output always leaves its sum (to an FFN stage, which completes it in
its input), and when its output declaration permits deferring the sum. The
consumer decides whether to fuse its completion with the input norm."""
__slots__ = (
"skips_reduction",
"_hands_on",
"_scope",
"_update",
"_declared_sum",
"_stream",
)
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
produced = communicator.plan._batch_steps(forward_batch).ffn_output
self._stream = None
self._update = produced.update
self._declared_sum = produced.group if produced.always_leaves else None
self._hands_on = (
produced.leaves_for_next_layer
and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
)
self.skips_reduction = produced.always_leaves or self._hands_on
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
def __enter__(self) -> MixerExit:
self._scope.__enter__()
return self
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(self, hidden_states: torch.Tensor, residual=None):
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
if self._stream is not None:
residual = self._stream
if self._hands_on:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
output = residual.leave(
hidden_states, self._update, declared_sum=self._declared_sum
)
return output if self._stream is not None else (output, residual)
return hidden_states
class FfnExit:
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
``defer_moe_finalize`` on ``get_forward()``."""
__slots__ = (
"communicator",
"forward_batch",
"defer_moe_finalize",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_complete",
"_update",
"_declared_sum",
"_scope",
"_stream",
)
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
self._stream = None
self.communicator = communicator
self.forward_batch = forward_batch
completion = communicator._select_ffn_completion(forward_batch)
self.defer_moe_finalize = completion.defer_moe_finalize
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
self._complete = completion.complete
steps = communicator.plan._batch_steps(forward_batch)
self._update = steps.ffn_output.update
self._declared_sum = communicator._declared_ffn_sum(
steps, completion.mlp_reduce_scatter
)
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
defer_moe_finalize=self.defer_moe_finalize,
)
def __enter__(self) -> FfnExit:
self._scope.__enter__()
return self
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input, or postprocess."""
if self._stream is not None:
residual = self._stream
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
residual = stream.residual
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
else:
hidden_states, residual = self._complete(hidden_states, residual)
result = self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update, self._declared_sum
)
return result[0] if self._stream is not None else result
@@ -0,0 +1,705 @@
"""Construct one computing stage and its transport from a predecessor contract."""
from __future__ import annotations
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Mapping, Optional
import msgspec
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.communicator.boundary import (
EdgeDecl,
StageDecl,
StageInput,
StageKind,
StageOutput,
_cp_moves,
)
from sglang.srt.layers.communicator.construction import (
BatchVariant,
StageEdges,
StagePlan,
_input_can_be_scattered,
_refuse_uncovered_cp_moe,
_use_ag_after_qlora,
)
from sglang.srt.layers.communicator.layout import (
Layout,
SumGroup,
TokenAxis,
_gathers_over_attention_cp,
_generic_prefill_cp_shards_tokens,
enable_moe_dense_fully_dp,
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.ops import (
_hand_qkv_hook_its_input,
_hand_scattered_input_to_attention,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
NORM_QUANT_READ,
NORM_READ,
)
from sglang.srt.layers.communicator.stage import StageCommunicator
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
def _variants(*, ordinary_only=False):
yield BatchVariant.ORDINARY
if ordinary_only:
return
if _generic_prefill_cp_shards_tokens():
yield BatchVariant.CONTEXT_PARALLEL
if _input_can_be_scattered():
yield BatchVariant.INPUT_SCATTERED
if layernorm_sp.layernorm_sp_enabled():
yield BatchVariant.SEQUENCE_PARALLEL
def _rows(variant):
axes = token_axis_sizes(cp_active=variant is BatchVariant.CONTEXT_PARALLEL)
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, TokenAxis.ATTN_TP_SCATTER, axis_sizes=axes
)
return axes, attention, local, Layout.sharded_over(axis_sizes=axes)
def _ffn_decl(
variant,
*,
sparse,
terminal=False,
next_sparse=False,
output=None,
read=NORM_READ,
update=ADD,
local_dense=True,
):
parallel = get_parallel()
reduction = get_exec().comm.boundary_reduction
if reduction not in ("ar", "rs", "rsv", "rs+rsv"):
raise ValueError(
"boundary_reduction must be resolved before model construction"
)
can_move_output = output is None or output.before_reduce_scatter
use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output
use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output
cp_shards = _generic_prefill_cp_shards_tokens()
axes, attention, local, full = _rows(variant)
on_local = (
is_moe_input_scattered_across_dp_ranks()
if sparse
else (local_dense and enable_moe_dense_fully_dp())
)
if parallel.attn_cp_size > 1 and sparse:
_refuse_uncovered_cp_moe(on_local, cp_shards)
on_cp = (
sparse
and parallel.attn_cp_size > 1
and parallel.moe_dp_size == parallel.attn_cp_size
and not _gathers_over_attention_cp()
)
group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
if variant is BatchVariant.SEQUENCE_PARALLEL:
return (
StageDecl(
StageInput(
full,
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
read=read,
),
StageOutput(local, update=update, transform=output),
),
local,
local,
)
if variant is BatchVariant.INPUT_SCATTERED:
scattered_residual = update.at_producer
residual = local if scattered_residual else attention
returned = local if scattered_residual and not terminal else attention
return (
StageDecl(
StageInput(full, read=read),
StageOutput(
attention,
group=group,
leaves_for_reduce_scatter=use_reduce_scatter
and (scattered_residual or not terminal),
update=update,
transform=output,
),
),
residual,
returned,
)
if cp_shards and _cp_moves().reduce_scatter is not None:
may_leave = variant is not BatchVariant.CONTEXT_PARALLEL
may_scatter = True
elif cp_shards or parallel.attn_dp_size > 1:
may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp
else:
may_leave = may_scatter = True
rows = (
local
if on_local
else Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes
)
)
output = (
StageOutput(rows, update=update, transform=output)
if on_local
else StageOutput(
rows,
group=group,
leaves_for_next_layer=may_leave
and not terminal
and not update.at_producer
and update.can_defer_across_layers
and output is None,
leaves_for_reduce_scatter=use_reduce_scatter and may_scatter,
leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave,
update=update,
transform=output,
)
)
gathers_for_tbo = (
enable_moe_dense_fully_dp()
and get_exec().overlap.enable_two_batch_overlap
and not sparse
and bool(next_sparse)
)
returned = local if on_local and not terminal and not gathers_for_tbo else attention
return (
StageDecl(StageInput(rows, read=read), output),
local if on_local else attention,
returned,
)
@dataclass(frozen=True)
class StageDeclaration:
"""A computation and its input source, without an execution plan.
``previous`` consumes a producer's output. ``prepared_from`` reuses an
already-read input for a branch; the two sources are mutually exclusive.
Sources may be reconstructed locally, including across pipeline ranks.
"""
kind: StageKind
read: StageRead
update: StageUpdate
sparse: bool = False
terminal: bool = False
next_sparse: bool = False
output: Optional[OutputTransform] = None
ordinary_only: bool = False
# A mixer with its own exit publishes reduction decisions before compute.
mixer_exit: bool = False
next_kind: Optional[StageKind] = None
return_to_attention: bool = False
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
def __post_init__(self):
if self.previous is not None and self.prepared_from is not None:
raise ValueError("choose a previous output or a prepared branch input")
for source in (self.previous, self.prepared_from):
if source is not None and not isinstance(source, StageDeclaration):
raise TypeError("stage sources must be declarations")
@dataclass(frozen=True)
class StageConnection:
"""The producer's exit and consumer's entry for each batch variant.
Immutable, norm-free declarations. Independently constructed equivalent
connections work across pipeline partitions; object identity is irrelevant.
"""
producer: Optional[StageDeclaration]
consumer: Optional[StageDeclaration]
exits: Mapping[BatchVariant, EdgeDecl]
entries: Mapping[BatchVariant, EdgeDecl]
def __post_init__(self):
object.__setattr__(self, "exits", MappingProxyType(dict(self.exits)))
object.__setattr__(self, "entries", MappingProxyType(dict(self.entries)))
def declare_attn(
*,
previous: Optional[StageDeclaration] = None,
prepared_from: Optional[StageDeclaration] = None,
read=NORM_QUANT_READ,
update=ADD,
terminal=False,
ordinary_only=False,
mixer_exit=False,
next_kind=None,
):
return StageDeclaration(
StageKind.ATTENTION,
read,
update,
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
ordinary_only=ordinary_only,
mixer_exit=mixer_exit,
next_kind=next_kind,
)
def declare_ffn(
*,
previous: Optional[StageDeclaration] = None,
prepared_from: Optional[StageDeclaration] = None,
sparse=False,
read=NORM_READ,
update=ADD,
terminal=False,
next_sparse=False,
output=None,
ordinary_only=False,
return_to_attention=False,
):
return StageDeclaration(
StageKind.FFN,
read,
update,
sparse=sparse,
previous=previous,
prepared_from=prepared_from,
terminal=terminal,
next_sparse=next_sparse,
output=output,
ordinary_only=ordinary_only,
return_to_attention=return_to_attention,
)
def _resolve(stage, variant):
axes, attention, local, full = _rows(variant)
if stage.kind is StageKind.FFN:
if stage.update.at_producer:
if stage.sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
)
declaration, residual, returned = _ffn_decl(
variant,
sparse=stage.sparse,
terminal=stage.terminal,
next_sparse=stage.next_sparse,
output=stage.output,
read=stage.read,
update=stage.update,
local_dense=not stage.return_to_attention,
)
if stage.return_to_attention:
returned = attention
return declaration, residual, returned
sp = variant is BatchVariant.SEQUENCE_PARALLEL
scattered = variant is BatchVariant.INPUT_SCATTERED
gathers = (
frozenset({TokenAxis.ATTN_TP_SCATTER})
if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora)
else frozenset()
)
owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1
declaration = StageDecl(
StageInput(attention, gathers_itself=gathers, read=stage.read),
StageOutput(
local if sp else attention,
group=SumGroup.ATTN_TP if owes else None,
always_leaves=owes
and (not stage.mixer_exit or stage.next_kind is StageKind.FFN),
leaves_for_next_layer=owes
and stage.mixer_exit
and stage.next_kind is StageKind.ATTENTION,
update=stage.update,
),
)
return declaration, None, attention
def _connect(producer, consumer, *, residual_from=None):
"""Resolve producer and consumer declarations; None denotes an external endpoint.
A missing producer is the stack input; a missing consumer is a handoff
whose next read is bound separately (or the terminal output).
``residual_from`` names the boundary that placed the producer's residual.
It is needed for attention, whose residual can stay on finer rows than its
compute input. No executable stage participates in this construction.
"""
if producer is None and consumer is None:
raise ValueError("a boundary needs at least one declared side")
before, after = producer, consumer
ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None)
exits, entries = {}, {}
for variant in _variants(ordinary_only=ordinary_only):
_, attention, local, _ = _rows(variant)
if before is None:
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
owes = variant is BatchVariant.INPUT_SCATTERED
arrived = StageOutput(
rows,
group=SumGroup.TP if owes else None,
always_leaves=owes,
update=None,
)
residual = rows
capabilities = (True,)
else:
decl, during, returned = _resolve(before, variant)
if during is None:
if residual_from is not None:
if residual_from.consumer != producer:
raise ValueError("residual source must enter the producer")
during = residual_from.entries[variant].residual_to
elif before.mixer_exit:
during = attention
else:
raise ValueError(
"attention output needs its incoming residual placement"
)
exits[variant] = EdgeDecl(
decl.output, StageInput(returned), during, returned
)
if before.kind is StageKind.ATTENTION and not before.mixer_exit:
arrived, residual, capabilities = decl.output, during, ()
else:
owes = (
variant is BatchVariant.INPUT_SCATTERED
and not before.update.at_producer
)
carries = (before.mixer_exit or before.return_to_attention) and (
decl.output.always_leaves or decl.output.leaves_for_next_layer
)
arrived = StageOutput(
returned,
group=decl.output.group
if carries
else (SumGroup.TP if owes else None),
always_leaves=decl.output.always_leaves if carries else owes,
leaves_for_next_layer=decl.output.leaves_for_next_layer
if carries
else False,
update=None,
)
residual, capabilities = returned, (before.update.adds_plainly,)
if after is None:
continue
decl, during, _ = _resolve(after, variant)
if after.kind is StageKind.ATTENTION:
during = (
Layout(residual.sharded | {TokenAxis.ATTN_TP_SCATTER})
if variant is BatchVariant.INPUT_SCATTERED
and arrived.always_leaves
and arrived.update is None
and not after.mixer_exit
else residual
)
elif after.return_to_attention:
during = (
decl.input.layout
if residual.sharded <= decl.input.layout.sharded
else residual
)
joins = (
variant is BatchVariant.INPUT_SCATTERED
and arrived.update is not None
and arrived.update.adds_plainly
and after.kind is StageKind.FFN
)
edge = EdgeDecl(
arrived,
decl.input,
residual,
during,
residual_joins_sum=joins,
update_capabilities=capabilities,
)
entries[variant] = edge
if (
before is not None
and before.kind is StageKind.ATTENTION
and not before.mixer_exit
):
exits[variant] = edge
return StageConnection(producer, consumer, exits, entries)
def _fork_input(prepared, consumer):
"""Place an already-read input onto a branch's computation rows.
The branch adapter performs the move and forks the stream. It does not
execute the branch's input norm again.
"""
entries = {}
for variant in _variants(ordinary_only=consumer.ordinary_only):
source = prepared.entries[variant]
declaration, during, _ = _resolve(consumer, variant)
entries[variant] = EdgeDecl(
StageOutput(source.need.layout, update=None),
declaration.input,
source.residual_to,
during,
update_capabilities=(True,),
)
return StageConnection(prepared.consumer, consumer, {}, entries)
def _incoming(stage):
if stage.prepared_from is not None:
return _fork_input(_incoming(stage.prepared_from), stage)
previous = stage.previous
# A normal attention may retain a finer residual than its output rows.
# FFN and mixer exits declare their returned residual placement directly.
source = (
_incoming(previous)
if previous is not None
and previous.kind is StageKind.ATTENTION
and not previous.mixer_exit
else None
)
return _connect(
previous,
stage,
residual_from=source,
)
def _connections(stage, following=None):
if following is not None and following.previous != stage:
raise ValueError("the following declaration must consume this stage's output")
incoming = _incoming(stage)
outgoing = _connect(
stage,
following,
residual_from=incoming,
)
return incoming, outgoing
def _bind_stage(declaration, norm, incoming, outgoing, **options):
if incoming.consumer != declaration or outgoing.producer != declaration:
raise ValueError("connections do not match the stage declaration")
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
if declaration.update.at_producer:
if declaration.sparse and sparse_moe_gathers_over_moe_cp():
raise NotImplementedError(
"MHC does not support a MoE gathered over the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
raise NotImplementedError(
"MHC with input-scattered attention under attention CP"
)
variants = {}
for variant, edge in incoming.entries.items():
if (
declaration.update.at_producer
and TokenAxis.ATTN_CP
in edge.produced.layout.sharded - edge.need.layout.sharded
):
raise NotImplementedError("MHC with a gather over attention CP")
handoff = None
if declaration.kind is StageKind.ATTENTION:
handoff = (
_hand_scattered_input_to_attention
if variant is BatchVariant.INPUT_SCATTERED
and not declaration.update.adds_plainly
else _hand_qkv_hook_its_input
)
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
plan = StagePlan(
declaration.kind,
norm,
variants,
enters_stack=incoming.producer is None,
prepared_input=declaration.prepared_from is not None,
terminal=declaration.terminal,
fixed_output=declaration.kind is StageKind.ATTENTION
and not declaration.mixer_exit,
is_sparse=declaration.sparse,
**options,
)
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
# Layout eligibility comes from the connected consumer, not a mutable
# link to its execution plan. Kernel binding remains consumer-owned.
from sglang.srt.layers.communicator.boundary import make_boundary
plan._fusion_rows = {
v: make_boundary(
edge,
cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None,
force_layernorm_before_gather=options.get(
"force_layernorm_before_dp_gather", False
),
).input_rows
for v, edge in outgoing.entries.items()
}
return StageCommunicator(plan, declaration.kind, norm, declaration=declaration)
def make_attn_stage(
*,
declaration,
norm,
following: Optional[StageDeclaration] = None,
qkv_latent_func=None,
force_layernorm_before_dp_gather=False,
enable_fused_ar_quant=False,
fused_ar_quant_keep_bf16=False,
residual_in_hidden=False,
fusions=None,
):
"""Resolve this attention's boundaries, then bind its norm and hooks.
``following`` supplies the consumer declaration for a stage inside a
layer. With no following stage, the output is handed across a layer or
stack boundary; the receiver binds its own read independently.
"""
if declaration.kind is not StageKind.ATTENTION:
raise TypeError("make_attn_stage requires an attention declaration")
incoming, outgoing = _connections(declaration, following)
return _bind_stage(
declaration,
norm,
incoming,
outgoing,
qkv_latent_func=qkv_latent_func,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
residual_in_hidden=residual_in_hidden,
fusions=fusions,
)
def make_ffn_stage(
*,
declaration,
norm,
following: Optional[StageDeclaration] = None,
force_layernorm_before_dp_gather=False,
fusions=None,
):
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
if declaration.kind is not StageKind.FFN:
raise TypeError("make_ffn_stage requires an FFN declaration")
incoming, outgoing = _connections(declaration, following)
return _bind_stage(
declaration,
norm,
incoming,
outgoing,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
residual_in_hidden=declaration.update.at_producer,
fusions=fusions,
)
def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
"""Bind a local linear sequence and return its independent boundaries.
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
Options are the stage-specific keyword arguments of ``make_attn_stage``
or ``make_ffn_stage``; they are consumed only during construction.
Declarations in the sequence have no input source: this function connects
them in order. Only the final boundary is terminal when requested.
``previous`` describes an external producer's output. ``prepared_from``
describes an already-read branch input, obtainable from the source
boundary's immutable ``declaration``. Neither takes an executable boundary.
No sequence object or runtime routing is retained.
"""
if not stages:
raise ValueError("make_stages needs at least one stage")
if previous is not None and prepared_from is not None:
raise ValueError("choose a previous output or a prepared branch input")
declarations = []
bindings = []
for index, item in enumerate(stages):
if len(item) not in (2, 3):
raise ValueError("a stage needs (declaration, norm[, options])")
declaration, norm = item[:2]
if not isinstance(declaration, StageDeclaration):
raise TypeError("make_stages requires stage declarations")
if declaration.previous is not None or declaration.prepared_from is not None:
raise ValueError("pass input sources to make_stages, not its declarations")
if declaration.terminal:
raise ValueError("pass terminal to make_stages, not its declarations")
declaration = replace(
declaration,
previous=previous if index == 0 else declarations[-1],
prepared_from=prepared_from if index == 0 else None,
terminal=terminal and index == len(stages) - 1,
next_kind=stages[index + 1][0].kind
if declaration.mixer_exit and index + 1 < len(stages)
else declaration.next_kind,
)
declarations.append(declaration)
bindings.append((norm, item[2] if len(item) == 3 else {}))
boundaries = []
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
build = (
make_attn_stage
if declaration.kind is StageKind.ATTENTION
else make_ffn_stage
)
boundaries.append(
build(
declaration=declaration,
norm=norm,
following=declarations[index + 1]
if index + 1 < len(declarations)
else None,
**options,
)
)
for producer, consumer in zip(boundaries, boundaries[1:]):
if (
producer.kind is not StageKind.ATTENTION
or producer.declaration.mixer_exit
or consumer.kind is not StageKind.FFN
):
continue
for variant, steps in producer.plan._paths.items():
predicate = consumer.plan._paths[variant].ffn.preserves_residual
if predicate is not None:
producer.plan._paths[variant] = msgspec.structs.replace(
steps,
attention=msgspec.structs.replace(
steps.attention, capture_preserves_residual=predicate
),
)
producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY]
producer.plan._cp_steps = producer.plan._paths.get(
BatchVariant.CONTEXT_PARALLEL
)
producer.plan._input_scattered_steps = producer.plan._paths.get(
BatchVariant.INPUT_SCATTERED
)
producer.plan._sp_steps = producer.plan._paths.get(
BatchVariant.SEQUENCE_PARALLEL
)
return tuple(boundaries)
@@ -0,0 +1,148 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Select fused completion and read kernels for each consumer."""
from functools import partial
from typing import Callable, Optional, Tuple
import torch
from sglang.srt.layers.communicator.boundary import FusedMlpInput
from sglang.srt.layers.communicator.layout import SumGroup
from sglang.srt.layers.communicator.output import UnreducedOutput
from sglang.srt.layers.communicator.residual.add_norm import (
apply_aiter_all_reduce_fusion,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.moe import post_experts_reduction_group
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
def attention_fusions(plan) -> Tuple[Callable, ...]:
"""The fused kernels that complete what the previous layer left together
with the residual update and the input norm, in the order they are tried.
Each takes (owed, residual, forward_batch, post_residual_addition) and
returns None when it does not take the batch. They add the residual
plainly; the boundary tries them only when the update it writes in is a
plain add. A backend's come first."""
given = plan.fusions.attention_input(plan) if plan.fusions else ()
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
return given
plan._attn_input_fuses_quant = (
plan.enable_fused_ar_quant
and _use_aiter
and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group")
)
return (*given, partial(complete_attention_input, plan))
def complete_attention_input(
plan,
owed: UnreducedOutput,
residual: torch.Tensor,
forward_batch: ForwardBatch,
post_residual_addition: Optional[torch.Tensor],
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""Complete the sum the previous layer left, add it to the residual and
apply the input norm in one aiter or flashinfer kernel; None when the
kernel does not take this batch. The result is not quantized for
``quant_format``."""
if (
not isinstance(owed, UnreducedOutput)
# The kernel does not add it.
or post_residual_addition is not None
# The kernel reduces over the MoE output's group.
or owed.group is not post_experts_reduction_group()
):
return None
hidden_states = owed.partial
if not (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
):
return None
if plan._attn_input_fuses_quant:
# Falls back to AR+RMSNorm + separate quant internally when the
# fully-fused kernel cannot service the shape.
quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group(
hidden_states,
residual,
use_attn_tp_group=False,
keep_bf16=plan.fused_ar_quant_keep_bf16,
)
if quant_result is not None:
return quant_result
return plan.norm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=False
)
def ffn_fusions(plan) -> Tuple["FusedMlpInput", ...]:
"""The fused kernels that can take the attention -> FFN steps, in the
order they are tried. They add the residual plainly; the boundary tries
them only when the update it writes in is a plain add. A backend's come
first."""
given = plan.fusions.ffn_input(plan) if plan.fusions else ()
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
return given
return (
*given,
FusedMlpInput(
completes=SumGroup.ATTN_TP,
run=partial(complete_ffn_input, plan),
preserves_residual=(
flashinfer_preserves_residual
if isinstance(plan.norm, (RMSNorm, GemmaRMSNorm))
and type(plan.norm).forward_with_allreduce_fusion
in (
RMSNorm.forward_with_allreduce_fusion,
GemmaRMSNorm.forward_with_allreduce_fusion,
)
else None
),
),
)
def flashinfer_preserves_residual(value, forward_batch):
"""The CUDA wrapper returns a fresh residual, including backend fallback."""
return (
not _use_aiter
and get_parallel().attn_tp_size > 1
and apply_flashinfer_allreduce_fusion(value.shape[0])
)
def complete_ffn_input(
plan,
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""The attention-TP all-reduce, residual add and norm in one aiter or
flashinfer kernel; None when neither takes the batch."""
if not (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
):
return None
return plan.norm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=True
)
+27 -93
View File
@@ -42,8 +42,8 @@ from sglang.srt.layers.communicator.boundary import (
sequence_parallel_layer_sides,
with_residual,
)
from sglang.srt.layers.communicator.exit import _reduce_and_redistribute_output_step
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -56,6 +56,7 @@ from sglang.srt.layers.communicator.layout import (
sparse_moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.communicator.legacy_stage import StageCommunicator
from sglang.srt.layers.communicator.ops import (
CommunicateSimpleFn,
_all_reduce_then_to_local_tokens,
@@ -63,7 +64,6 @@ from sglang.srt.layers.communicator.ops import (
_hand_scattered_input_to_attention,
_mlp_input_without_dp,
_redistribute_output,
_reduce_and_redistribute_output_step,
_to_local_tokens,
move_rows,
)
@@ -189,79 +189,6 @@ def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool
)
class StageCommunicator:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
self._layer = layer
self._name = name
self._norm_name = norm_name
@property
def norm(self) -> torch.nn.Module:
"""The norm the stage reads its input with, one of the layer's."""
return getattr(self._layer, self._norm_name)
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
) -> StageEntry:
if steps is None:
steps = self._layer._batch_steps(forward_batch)
entry = getattr(steps, self._name)
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self._name} stage"
)
return entry
def prepare(
self,
hidden_states,
residual,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
context = self._layer._context
stream = residual_batch.current(forward_batch)
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
if (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
):
hidden_states = stream.leave(
hidden_states, ADD, declared_sum=entry.input_sum
)
if stream.pending is not None:
call["update"] = stream.pending.update
call["pending"] = stream.pending
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, context, **call
)
if entry.input_move is not None:
hidden_states = entry.input_move(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=context,
)
if entry.handoff is not None:
hidden_states = entry.handoff(
hidden_states, forward_batch, self._layer.qkv_latent_func
)
stream.write(residual)
return hidden_states, stream
class LayerFusions(Protocol):
"""The fused kernels a fusion backend gives a layer, bound to it and tried
before the layer's own: at its attention input (each takes what the previous
@@ -336,10 +263,6 @@ class LayerCommunicator:
self._residual = residual
self.fusions = fusions
self._context = CommunicateContext.init_new()
self._context.force_layernorm_before_dp_gather = (
force_layernorm_before_dp_gather
)
# The fused kernels every batch's attention input tries first.
self._attn_input_fusions = self._select_attn_input_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
@@ -799,7 +722,6 @@ class LayerCommunicator:
else move(
hidden_states=residual_value,
forward_batch=forward_batch,
context=self._context,
)
)
keeps = self.attn.entry(forward_batch).capture_preserves_residual
@@ -850,7 +772,7 @@ class LayerCommunicator:
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
return self.attn.prepare(
return self.attn._prepare(
hidden_states,
residual,
forward_batch,
@@ -979,8 +901,6 @@ class LayerCommunicator:
):
steps = self._batch_steps(forward_batch)
self.publish_mlp_lora_layout(steps)
if cache is not None:
self._context.cache = cache
if isinstance(residual, ResidualStream) and self.stage_edges is None:
# The paired boundary records the attention contribution once.
@@ -989,12 +909,13 @@ class LayerCommunicator:
self._residual.attention_update,
declared_sum=steps.ffn.input_sum,
)
return self.ffn.prepare(
return self.ffn._prepare(
hidden_states,
residual,
forward_batch,
steps,
update=self._residual.attention_update,
cache=cache,
)
def maybe_prefetch_next_full_attention_kv(
@@ -1180,7 +1101,6 @@ class LayerCommunicator:
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_facts.is_layer_sparse,
)
@@ -1317,7 +1237,7 @@ class LayerCommunicator:
input when no fused kernel takes it: when the next layer would run the
same all-reduce the FFN itself would have."""
return (
self._context.tp_size > 1
get_parallel().tp_size > 1
and not self.is_last_layer
and self._ffn_sum_can_move_to_next_layer(forward_batch)
and _can_defer_ffn_reduction(forward_batch, self)
@@ -1361,15 +1281,23 @@ class MixerExit:
its input), and when its output declaration permits deferring the sum. The
consumer decides whether to fuse its completion with the input norm."""
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum")
__slots__ = (
"skips_reduction",
"_hands_on",
"_scope",
"_update",
"_declared_sum",
"_stream",
)
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
produced = communicator._batch_steps(forward_batch).ffn_output
self._stream = None
self._update = produced.update
self._declared_sum = produced.group if produced.always_leaves else None
self._hands_on = (
produced.leaves_for_next_layer
and communicator._context.tp_size > 1
and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
)
@@ -1386,14 +1314,17 @@ class MixerExit:
def finish(self, hidden_states: torch.Tensor, residual=None):
"""The mixer's output: its partial sum to the FFN stage after it, as a
value that owes the sum to a mixer after it, or complete."""
if self._stream is not None:
residual = self._stream
if self._hands_on:
hidden_states = UnreducedOutput(
hidden_states, group=get_parallel().tp_group
)
if isinstance(residual, ResidualStream):
return residual.leave(
output = residual.leave(
hidden_states, self._update, declared_sum=self._declared_sum
), residual
)
return output if self._stream is not None else (output, residual)
return hidden_states
@@ -1412,9 +1343,11 @@ class FfnExit:
"_update",
"_declared_sum",
"_scope",
"_stream",
)
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
self._stream = None
self.communicator = communicator
self.forward_batch = forward_batch
completion = communicator._select_ffn_completion(forward_batch)
@@ -1441,7 +1374,7 @@ class FfnExit:
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input, or postprocess."""
stream = residual_batch.current(self.forward_batch)
@@ -1452,9 +1385,10 @@ class FfnExit:
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
else:
hidden_states, residual = self._complete(hidden_states, residual)
return self.communicator._leave_ffn_output(
result = self.communicator._leave_ffn_output(
hidden_states, residual, stream, self._update, self._declared_sum
)
return result[0] if self._stream is not None else result
class MHCLayerCommunicator(LayerCommunicator):
@@ -1501,7 +1435,7 @@ class MHCLayerCommunicator(LayerCommunicator):
residual=self.mhc.layer_residual(),
)
# MHC has not been run with input-scattered attention under attention CP.
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
if get_parallel().attn_cp_size > 1 and self._input_can_be_scattered():
raise NotImplementedError(
"MHCLayerCommunicator with input-scattered attention under attention CP"
)
@@ -14,7 +14,6 @@
"""Token axes, layouts and process groups of what a layer hands across its
boundaries, and the configuration and batch facts they depend on."""
from dataclasses import dataclass
from enum import Enum, auto
from typing import Dict, FrozenSet, List, Mapping, Optional
@@ -94,38 +93,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
return _batch_size(forward_batch) > 0
@dataclass
class CommunicateContext:
attn_tp_rank: int
attn_tp_size: int
attn_dp_size: int
attn_cp_rank: int
attn_cp_size: int
tp_size: int
cache = None
tp_rank: int
force_layernorm_before_dp_gather: bool = False
@classmethod
def init_new(cls):
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
attn_dp_size = get_parallel().attn_dp_size
attn_cp_size = get_parallel().attn_cp_size
attn_cp_rank = get_parallel().attn_cp_rank
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
return cls(
attn_tp_rank=attn_tp_rank,
attn_tp_size=attn_tp_size,
attn_dp_size=attn_dp_size,
attn_cp_rank=attn_cp_rank,
attn_cp_size=attn_cp_size,
tp_size=tp_size,
tp_rank=tp_rank,
)
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
gathered across the MoE-CP group, or None when it is not: only a context
@@ -0,0 +1,160 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A stage's model-facing input and output boundary."""
from typing import TYPE_CHECKING, Optional
from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry
from sglang.srt.layers.communicator.residual.add_norm import ADD
from sglang.srt.layers.communicator.residual.batch import current
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
if TYPE_CHECKING:
from sglang.srt.layers.communicator.layer import LayerCommunicator
class StageCommunicator:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
self._layer = layer
self._name = name
self.norm = getattr(layer, norm_name)
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
) -> StageEntry:
if steps is None:
steps = self._layer._batch_steps(forward_batch)
entry = getattr(steps, self._name)
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self._name} stage"
)
return entry
def _prepare(
self,
hidden_states,
residual,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
stream = current(forward_batch)
if residual is not stream:
raise RuntimeError("residual alias belongs to a different invocation")
if (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
):
hidden_states = stream.leave(
hidden_states, ADD, declared_sum=entry.input_sum
)
if stream.pending is not None:
call["update"] = stream.pending.update
call["pending"] = stream.pending
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, **call
)
if entry.input_move is not None:
hidden_states = entry.input_move(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
if entry.handoff is not None:
hidden_states = entry.handoff(
hidden_states, forward_batch, self._layer.qkv_latent_func
)
stream.write(residual)
return hidden_states, stream
@property
def input_rows(self):
return self._layer.input_rows
@property
def input_on_attention_tp_slices(self):
return self._layer.input_on_attention_tp_slices
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
stream = current(forward_batch)
if self._name == "attention":
hidden_states, stream = (
self._layer.prepare_attn_and_capture_last_layer_outputs(
hidden_states, stream, forward_batch, **call
)
)
else:
steps = self._layer._batch_steps(forward_batch)
self._layer.publish_mlp_lora_layout(steps)
if cache is not None:
call["cache"] = cache
hidden_states, stream = self._prepare(
hidden_states, stream, forward_batch, steps, **call
)
forward_batch.residual_stream = stream
return hidden_states
def finish(self, hidden_states, forward_batch):
"""Hand this attention's actual contribution to the following stage."""
if self._name != "attention" or self._layer.stage_edges is not None:
raise RuntimeError(
"use the stage exit scope for an FFN or single-stage mixer"
)
return current(forward_batch).leave(
hidden_states,
self._layer._residual.attention_update,
declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum,
)
def exit(self, forward_batch):
result = (
self._layer.ffn_exit(forward_batch)
if self._name == "ffn"
else self._layer.mixer_exit(forward_batch)
)
result._stream = current(forward_batch)
return result
def from_pp(self, tensors, forward_batch, **kwargs):
hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs)
forward_batch.residual_stream = stream
return hidden_states
def snapshot(self, hidden_states, forward_batch, *, at_input=False):
return self._layer.snapshot(
hidden_states, current(forward_batch), at_input=at_input
)
def capture_output(self, hidden_states, forward_batch, **kwargs):
return self._layer.capture_output(
hidden_states, current(forward_batch), **kwargs
)
def postprocess(self, hidden_states, forward_batch):
hidden_states, stream = self._layer.postprocess_layer(
hidden_states, current(forward_batch), forward_batch
)
forward_batch.residual_stream = stream
return hidden_states
+58 -112
View File
@@ -38,7 +38,6 @@ from sglang.srt.layers.communicator.adapters.context_parallel import (
dsa_cp_reduce_scatter_hidden_states,
)
from sglang.srt.layers.communicator.layout import (
CommunicateContext,
Layout,
SumGroup,
TokenAxis,
@@ -56,7 +55,6 @@ from sglang.srt.layers.communicator.residual.stream import DeclaredSum
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
can_use_dp_reduce_scatter,
dp_gather_partial,
dp_gather_replicate,
dp_reduce_scatter_tensor,
@@ -69,7 +67,6 @@ from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
moe_cp_all_gather_into_tensor,
)
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
@@ -86,28 +83,30 @@ if _is_npu:
def tp_reduce_scatter(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
context: "CommunicateContext",
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Module-level so MHC communicators can reuse it without a
``LayerCommunicator`` instance."""
"""Complete a TP sum onto each rank's token slice."""
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
assert hidden_states.shape[0] % context.tp_size == 0, (
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
assert hidden_states.shape[0] % parallel.tp_size == 0, (
f"Expected total tokens {hidden_states.shape[0]} % tp_size {parallel.tp_size} to be 0"
)
local_tokens = hidden_states.shape[0] // context.tp_size
local_tokens = hidden_states.shape[0] // parallel.tp_size
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
parallel.tp_group.reduce_scatter_tensor(output, hidden_states)
if residual is not None:
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
return output, residual
def tp_slice(hidden_states, residual, context):
def tp_slice(hidden_states, residual):
"""The rows reduce-scatter would return, when the sum is already complete."""
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone()
parallel = get_parallel()
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
parallel.tp_rank
].clone()
if residual is not None:
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
return hidden_states, residual
@@ -116,7 +115,6 @@ class CommunicateSimpleFn:
def _trivial(
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
) -> torch.Tensor:
return hidden_states
@@ -124,18 +122,18 @@ class CommunicateSimpleFn:
def _scattered_to_tp_attn_full(
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
forward_batch: ForwardBatch,
context: CommunicateContext,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
parallel = get_parallel()
if isinstance(hidden_states, tuple):
gathered_hidden_states = []
for local_hidden_states in hidden_states:
with use_symmetric_memory(
get_parallel().tp_group,
parallel.tp_group,
disabled=not is_allocation_symmetric(),
):
output = torch.empty(
(
local_hidden_states.shape[0] * context.attn_tp_size,
local_hidden_states.shape[0] * parallel.attn_tp_size,
*local_hidden_states.shape[1:],
),
dtype=local_hidden_states.dtype,
@@ -152,10 +150,11 @@ class CommunicateSimpleFn:
def _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states: torch.Tensor, context: CommunicateContext
hidden_states: torch.Tensor,
) -> torch.Tensor:
local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
context.attn_tp_rank
parallel = get_parallel()
local_hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
parallel.attn_tp_rank
]
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
return local_hidden_states
@@ -250,8 +249,8 @@ def _mlp_input_without_dp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
gathers_residual: bool,
fusions: Tuple[Callable, ...],
group: SumGroup = SumGroup.ATTN_TP,
@@ -274,8 +273,8 @@ def _mlp_input_without_dp(
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if _is_npu and context.cache is not None:
_ = prepare_weight_cache(hidden_states, context.cache)
if _is_npu and cache is not None:
_ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -284,8 +283,8 @@ def _mlp_input_dp_replicate(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
@@ -324,8 +323,8 @@ def _mlp_input_dp_partial(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
gathers_residual: bool,
places_cp_shards: bool = False,
read: StageRead = NORM_READ,
@@ -336,7 +335,7 @@ def _mlp_input_dp_partial(
shard of the DP group's tokens beside the others' in the group's slot."""
if gathers_residual:
residual = _redistribute_from_attn_tp_shards(residual)
if context.attn_tp_rank == 0:
if get_parallel().attn_tp_rank == 0:
hidden_states += residual
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
hidden_states = _reduce_and_redistribute_output_to_dp(
@@ -395,7 +394,7 @@ def _tp_all_gather_scattered_rows(
def _dispatch_consumer(
hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
):
if residual is None:
# Enter/read-only paths do not execute an update.
@@ -405,9 +404,7 @@ def _dispatch_consumer(
prepare = paths[update.adds_plainly]
except KeyError:
raise RuntimeError("producer update has no bound input path") from None
return prepare(
hidden_states, residual, forward_batch, norm, context, update=update, **call
)
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
def _consumer_step(
@@ -415,7 +412,6 @@ def _consumer_step(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
context: CommunicateContext,
*,
step: Callable,
adds_plainly: bool,
@@ -473,9 +469,7 @@ def _consumer_step(
hidden_states = reduce_output(owed)
else:
hidden_states = owed.partial
return step(
hidden_states, residual, forward_batch, norm, context, update=update, **call
)
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
def _read_input(
@@ -483,8 +477,8 @@ def _read_input(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
layer_input: Optional[Callable],
enters_stack: bool,
read: StageRead,
@@ -498,7 +492,7 @@ def _read_input(
starts its residual from its input."""
enters = residual is None and enters_stack
if layer_input is not None:
hidden_states, residual = layer_input(hidden_states, residual, context)
hidden_states, residual = layer_input(hidden_states, residual)
if enters:
hidden_states, residual = read.enter(hidden_states), None
if residual is None:
@@ -519,17 +513,15 @@ def _mlp_input_scatter(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
hidden_states, context
)
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
if scatters_residual:
residual = update.residual_to_attn_tp_shard(residual, context)
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -538,15 +530,15 @@ def _mlp_input_slice(
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
scatters_residual: bool,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
if scatters_residual and residual is not None:
residual = update.residual_to_attn_tp_shard(residual, context)
residual = update.residual_to_attn_tp_shard(residual)
return read.update_and_read(update, hidden_states, residual, layernorm)
@@ -555,8 +547,8 @@ def _mlp_input_on_residual_shard(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
reduces: bool = True,
read: StageRead,
update: StageUpdate,
@@ -565,11 +557,12 @@ def _mlp_input_on_residual_shard(
rows: reduce-scatter the attention output onto the slice, which completes
its sum, write it into the residual and read the FFN input there, then
gather the input back into the attention output's rows."""
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
if reduces:
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
else:
shard = shard.clone()
shard, residual = read.update_and_read(update, shard, residual, layernorm)
@@ -582,13 +575,13 @@ def _mlp_input_residual_into_sum(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
read: StageRead = NORM_READ,
update: StageUpdate = ADD,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context, read
hidden_states, residual, layernorm, read
)
@@ -596,13 +589,13 @@ def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
layernorm: torch.nn.Module,
context: CommunicateContext,
read: StageRead = NORM_READ,
):
parallel = get_parallel()
if hidden_states.shape[0] == 0:
return hidden_states, hidden_states
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
scattered_states += residual
residual = tensor_model_parallel_all_reduce(hidden_states)
return read.read(residual, layernorm)
@@ -613,8 +606,8 @@ def _mlp_input_gather_attention_cp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
@@ -622,7 +615,7 @@ def _mlp_input_gather_attention_cp(
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context, update=update
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
return dsa_cp_gather_hidden_states(hidden_states), residual
@@ -632,8 +625,8 @@ def _mlp_input_gather_moe_cp(
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
*,
cache=None,
gather: Callable,
update: StageUpdate = ADD,
):
@@ -649,7 +642,7 @@ def _mlp_input_gather_moe_cp(
return hidden_states, residual
hidden_states, residual = gather(
hidden_states, residual, forward_batch, layernorm, context, update=update
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
)
rows = moe_cp_gathered_rows(forward_batch)
@@ -709,27 +702,6 @@ def _redistribute_output_from_moe_cp(
).contiguous()
def _reduce_and_redistribute_output_step(
forward_batch: ForwardBatch,
*,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
"""The reduce-scatter that brings an FFN output gathered over attention
DP back to this rank's tokens when the FFN leaves its sum to it (see
StageOutput); None when the FFN reduces the output and only a scatter
remains."""
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
return _reduce_and_redistribute_output_varlen
if (
leaves_for_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
return _reduce_and_redistribute_output_max_len
return None
def _to_local_tokens(
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
forward_batch: ForwardBatch,
@@ -756,38 +728,15 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
return hidden_states, residual
@staticmethod
def _scatter_hidden_states(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
allow_reduce_scatter: bool = False,
is_layer_sparse: bool = False,
):
step = (
_reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=allow_reduce_scatter,
# A MoE block leaves its sum to reduce_scatterv whenever it
# applies (should_skip_post_experts_all_reduce).
leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
)
or _redistribute_output
)
return _to_local_tokens(step, forward_batch, hidden_states), residual
@staticmethod
def _gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
update: StageUpdate = ADD,
**kwargs,
):
@@ -799,7 +748,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
*,
sums: bool,
gathers_back: bool,
@@ -810,17 +758,20 @@ class CommunicateSummableTensorPairFn:
a reduce-scatter that also completes its sum when ``sums``, else this
rank's slice of the complete output. With ``gathers_back`` it is
written into the residual there and the full rows are gathered back."""
parallel = get_parallel()
if sums:
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
else:
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
parallel.tp_rank
]
if not gathers_back:
return hidden_states, residual
local_states = update.update(hidden_states, residual)
hidden_states = local_states.new_empty(
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
)
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
return hidden_states, None
@staticmethod
@@ -828,17 +779,15 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
):
assert residual is None, "not yet handled residual!=None"
return _redistribute_to_attn_tp_shards(hidden_states, context), None
return _redistribute_to_attn_tp_shards(hidden_states), None
@staticmethod
def _take_back_attention_cp_shard(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: this rank's shard of a complete output gathered in
@@ -852,7 +801,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
@@ -864,7 +812,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""This rank's CP shard of the rows the DP gather put in its DP group's
@@ -886,7 +833,6 @@ class CommunicateSummableTensorPairFn:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
context: CommunicateContext,
**kwargs,
):
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
@@ -905,7 +851,7 @@ class CommunicateSummableTensorPairFn:
if rows is not None:
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
if context.attn_dp_size > 1:
if get_parallel().attn_dp_size > 1:
hidden_states = _to_local_tokens(
_redistribute_output, forward_batch, hidden_states
)
@@ -21,6 +21,18 @@ import torch
from sglang.srt.distributed import GroupCoordinator
class OutputTransform(msgspec.Struct, frozen=True):
"""An operation on the producer contribution, before residual addition.
Some existing implementations apply it to a partial before reduce-scatter
but to a complete value after all-reduce. These are explicit alternatives,
not an assumption that an arbitrary operation commutes with reduction.
"""
apply: Callable[[torch.Tensor], torch.Tensor]
before_reduce_scatter: bool = False
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its sum, left for the next layer's input.
Hand it to the next layer, or pass it through reduce_output() before reading
@@ -38,7 +38,7 @@ class StageUpdate(Protocol):
def update(self, hidden_states, residual) -> torch.Tensor:
"""The residual with the output written into it."""
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
def residual_to_attn_tp_shard(self, residual) -> torch.Tensor:
"""This attention-TP rank's slice of the residual, with whatever the
update reads along with it."""
@@ -105,7 +105,7 @@ def finish_layer_stack(
return reduce_output(hidden_states), residual
def norm_output(hidden_states, residual, norm, capture_output=None):
def norm_output(hidden_states, residual, norm, capture_output=None, **read_kwargs):
"""The final norm and an optional capture of the same updated residual.
Keep add and norm together: normalizing a separately rounded residual is
@@ -116,7 +116,7 @@ def norm_output(hidden_states, residual, norm, capture_output=None):
if capture_output is not None:
capture_output(hidden_states)
return norm(hidden_states)
hidden_states, residual = norm(hidden_states, residual)
hidden_states, residual = norm(hidden_states, residual, **read_kwargs)
if capture_output is not None:
capture_output(residual)
return hidden_states
@@ -273,8 +273,8 @@ class Add:
hidden_states += residual
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
return _redistribute_to_attn_tp_shards(residual, context)
def residual_to_attn_tp_shard(self, residual):
return _redistribute_to_attn_tp_shards(residual)
def residual_from_attn_tp_shards(self, residual):
return _redistribute_from_attn_tp_shards(residual)
@@ -1,18 +1,82 @@
"""Own one invocation's residual stream, including independent TBO children."""
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Residual access for one layer-stack invocation or TBO microbatch."""
from sglang.srt.layers.communicator.residual import access
from sglang.srt.layers.communicator.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
def start(forward_batch):
"""Begin a fresh embedding-side invocation and return its stream."""
stream = ResidualStream()
forward_batch.residual_stream = stream
return stream
def current(forward_batch):
"""Return the invocation's authoritative stream while its stack is active."""
stream = forward_batch.residual_stream
if stream is None:
raise RuntimeError("start the layer stack before entering a stage")
return stream
def finish(hidden_states, forward_batch):
return current(forward_batch).complete(hidden_states)
def take_output(hidden_states, forward_batch):
"""Release a terminal output whose residual update has already been written."""
stream = current(forward_batch)
stream.check(hidden_states)
if stream.pending is not None:
raise RuntimeError("write the residual update before taking the final output")
forward_batch.residual_stream = None
return hidden_states
def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs):
hidden_states, residual = current(forward_batch).finish(hidden_states)
# The terminal consumer now owns the pair. Do not keep layer buffers alive
# through logits processing or the next forward on this batch.
forward_batch.residual_stream = None
return access.norm_output(
hidden_states, residual, layernorm, capture_output, **read_kwargs
)
def to_pp(hidden_states, forward_batch, *, preserve_declared=True):
hidden_states, residual = current(forward_batch).finish(
hidden_states, preserve_declared=preserve_declared
)
forward_batch.residual_stream = None
return PPProxyTensors({"hidden_states": hidden_states, "residual": residual})
def snapshot(hidden_states, forward_batch):
return current(forward_batch).snapshot(hidden_states)
def add_to_output(hidden_states, forward_batch, extra):
output, _ = access.add_to_output(hidden_states, current(forward_batch), extra)
return output
def fold(hidden_states, forward_batch):
output, _ = access.fold(hidden_states, current(forward_batch))
return output
def written(hidden_states, forward_batch):
forward_batch.residual_stream = ResidualStream(hidden_states)
return hidden_states
@@ -20,6 +20,7 @@ import torch
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
from sglang.srt.layers.communicator.residual import LayerResidual
from sglang.srt.runtime_context import get_parallel
@dataclass
@@ -96,8 +97,9 @@ class MHCState:
self.h_res = None
self.h_post = None
def residual_to_attn_tp_shard(self, residual, context):
rank, size = context.attn_tp_rank, context.attn_tp_size
def residual_to_attn_tp_shard(self, residual):
parallel = get_parallel()
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
self.h_res = self.h_res.tensor_split(size)[rank]
self.h_post = self.h_post.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
@@ -151,8 +153,8 @@ class _AttentionUpdate:
def update(self, hidden_states, residual):
return self.state.mlp_combine(hidden_states, residual)
def residual_to_attn_tp_shard(self, residual, context):
return self.state.residual_to_attn_tp_shard(residual, context)
def residual_to_attn_tp_shard(self, residual):
return self.state.residual_to_attn_tp_shard(residual)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
@@ -198,8 +200,8 @@ class _FfnUpdate:
hidden_states = hc_contract(hidden_states, self.state.hc_mult)
return hidden_states
def residual_to_attn_tp_shard(self, residual, context):
return self.state.residual_to_attn_tp_shard(residual, context)
def residual_to_attn_tp_shard(self, residual):
return self.state.residual_to_attn_tp_shard(residual)
def residual_from_attn_tp_shards(self, residual):
return self.state.residual_from_attn_tp_shards(residual)
@@ -0,0 +1,329 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A stage's model-facing input and output boundary."""
from typing import TYPE_CHECKING, Callable, Optional
import torch
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.communicator.adapters import branch
from sglang.srt.layers.communicator.adapters.lora import (
publish_attention,
publish_ffn,
)
from sglang.srt.layers.communicator.boundary import (
BoundarySteps,
StageEntry,
StageKind,
)
from sglang.srt.layers.communicator.ops import CommunicateSimpleFn
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
from sglang.srt.layers.communicator.residual.add_norm import (
ADD,
)
from sglang.srt.layers.communicator.residual.batch import current
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_forward
if TYPE_CHECKING:
from sglang.srt.layers.communicator.construction import StagePlan
from sglang.srt.layers.communicator.factories import StageDeclaration
class StageCommunicator:
"""One stage of a layer, its attention or its FFN: the boundary into it,
run with the stage's norm on the steps the layer chose for the batch."""
def __init__(
self,
plan: "StagePlan",
kind: StageKind,
norm,
*,
declaration: Optional["StageDeclaration"] = None,
):
self.plan = plan
self.kind = kind
self.norm = norm
self._declaration = declaration
@property
def declaration(self) -> "StageDeclaration":
"""Immutable source contract for constructing another local sequence."""
if self._declaration is None:
raise RuntimeError("this boundary was constructed from raw layout edges")
return self._declaration
def entry(
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
) -> StageEntry:
if steps is None:
steps = self.plan._batch_steps(forward_batch)
entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn
if entry is None:
raise NotImplementedError(
f"a layer that is one stage has no {self.kind.name.lower()} stage"
)
return entry
def _prepare(
self,
hidden_states,
residual,
forward_batch: ForwardBatch,
steps: Optional[BoundarySteps] = None,
**call,
):
"""The stage's input and the residual, from the previous stage's output
(``call``: what the stage's read takes, e.g. the attention's
``quant_format``)."""
entry = self.entry(forward_batch, steps)
stream = (
residual
if isinstance(residual, ResidualStream)
else ResidualStream(residual)
)
if not isinstance(residual, ResidualStream):
if residual is not None or entry.input_sum is not None:
hidden_states = stream.leave(
hidden_states, call.get("update", ADD), declared_sum=entry.input_sum
)
elif (
stream.pending is None
and stream.residual is None
and entry.input_sum is not None
):
hidden_states = stream.leave(
hidden_states, ADD, declared_sum=entry.input_sum
)
if stream.pending is not None:
call["update"] = stream.pending.update
call["pending"] = stream.pending
if stream.pending is None and stream.residual is not None:
call["written"] = True
hidden_states, residual = stream.input(hidden_states)
hidden_states, residual = entry.prepare(
hidden_states, residual, forward_batch, self.norm, **call
)
if entry.input_move is not None:
hidden_states = entry.input_move(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
if entry.handoff is not None:
hidden_states = entry.handoff(
hidden_states, forward_batch, self.plan.qkv_latent_func
)
stream.write(residual)
return hidden_states, stream
@property
def fusions(self):
return self.plan.fusions
@property
def input_rows(self):
return self.plan.input_rows
@property
def input_on_attention_tp_slices(self):
return self.plan.input_on_attention_tp_slices
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
if cache is not None:
call["cache"] = cache
prepare = (
self._prepare_attention
if self.kind is StageKind.ATTENTION
else self._prepare_input
)
hidden_states, forward_batch.residual_stream = prepare(
hidden_states, current(forward_batch), forward_batch, **call
)
return hidden_states
def finish(self, hidden_states, forward_batch):
"""Hand this attention's actual contribution to the following stage."""
if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output:
raise RuntimeError(
"use the stage exit scope for an FFN or single-stage mixer"
)
produced = self.plan.produced(self.kind, forward_batch)
return current(forward_batch).leave(
hidden_states,
produced.update,
declared_sum=produced.group if produced.always_leaves else None,
)
def exit(self, forward_batch):
result = (
self.plan.output.ffn_exit(forward_batch)
if self.kind is StageKind.FFN
else self.plan.output.mixer_exit(forward_batch)
)
result._stream = current(forward_batch)
return result
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
hidden_states, residual = from_pp(
tensors,
residual_in_hidden=self.plan.residual_in_hidden,
allow_missing_residual=allow_missing_residual,
)
declared_sum = None
if forward_batch is not None:
steps = self.plan._batch_steps(forward_batch)
entry = steps.attention or steps.ffn
declared_sum = entry.input_sum
hidden_states, forward_batch.residual_stream = ResidualStream.arrive(
hidden_states, residual, ADD, declared_sum=declared_sum
)
return hidden_states
def snapshot(self, hidden_states, forward_batch):
return current(forward_batch).snapshot(hidden_states)
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
stream = current(forward_batch)
if skip_empty:
storage = buffer(hidden_states)
if storage is not None and storage.shape[0] == 0:
return hidden_states, None
if stream.pending is None or not isinstance(stream.pending.owed, DeclaredSum):
hidden_states = stream.complete(hidden_states)
return hidden_states, stream.snapshot(hidden_states)
def postprocess(self, hidden_states, forward_batch):
hidden_states, stream = self.plan.output.postprocess_layer(
hidden_states, current(forward_batch), forward_batch
)
forward_batch.residual_stream = stream
return hidden_states
def branch_input(self, source, hidden_states, forward_batch):
hidden_states, stream = branch.branch_input(
self.plan,
source.plan,
hidden_states,
current(forward_batch),
forward_batch,
)
forward_batch.residual_stream = stream
return hidden_states
def branch_output(self, hidden_states, forward_batch):
return branch.branch_output(self.plan, hidden_states, forward_batch)
def merge_branch(self, contribution, hidden_states, source, forward_batch):
hidden_states, stream = branch.merge_branch(
self.plan,
contribution,
hidden_states,
current(forward_batch),
source.plan,
forward_batch,
)
forward_batch.residual_stream = stream
return hidden_states
def _prepare_input(self, hidden_states, residual, forward_batch, **call):
plan = self.plan
if self.kind is StageKind.ATTENTION:
publish_attention(plan._publish_lora_layout)
if plan._sp_steps is not None and plan.is_first_layer:
get_forward().set(
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
else:
publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows)
return self._prepare(hidden_states, residual, forward_batch, **call)
def _prepare_attention(
self,
hidden_states: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
post_residual_addition: Optional[torch.Tensor] = None,
quant_format: str = "",
capture_output: Optional[Callable] = None,
):
# Aux consumers need a materialized output before the input norm.
# Complete communication here, preserving the existing add+norm kernel
# and its FP32 accumulation. Its residual result also supplies capture.
capture_before_read = capture_output is not None and (
(stream.residual is None and stream.pending is None)
or post_residual_addition is not None
)
if capture_output is not None:
hidden_states = stream.complete(hidden_states)
if capture_before_read:
# Embeddings precede enter; HF deepstack capture precedes the
# extra addition. Neither is the residual returned by the read.
value, previous = stream.finish(hidden_states)
if previous is None:
capture_output(value)
else:
capture_output(value + previous, owned=True)
hidden_states, stream = self._prepare_input(
hidden_states,
stream,
forward_batch,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
entry = self.entry(forward_batch)
if capture_output is not None and not capture_before_read:
value = stream.residual
move = entry.capture_move
if move is not None:
value = move(value, forward_batch=forward_batch)
keeps_input = entry.capture_preserves_residual
capture_output(
value,
owned=entry.capture_move_allocates
or (
move is None
and keeps_input is not None
and keeps_input(value, forward_batch)
),
)
if captured_last_layer_outputs is not None:
residual_value = stream.residual
move = entry.input_move
gathered_last_layer_output = (
residual_value
if move is None
else move(
hidden_states=residual_value,
forward_batch=forward_batch,
)
)
# Input gathers allocate fresh DP buffers.
# Without a gather this is the mutable residual, so retention copies.
captured_last_layer_outputs.capture(
gathered_last_layer_output,
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
or (
move is None
and entry.capture_preserves_residual is not None
and entry.capture_preserves_residual(residual_value, forward_batch)
),
)
return hidden_states, stream
+8 -5
View File
@@ -559,7 +559,12 @@ def draft_model_build_scope():
original_scope = moe.in_speculative_scope
try:
moe.in_speculative_scope = True
yield
# Boundaries capture this resolved preference while the draft builds.
# Restoring it cannot change an already constructed target plan.
with get_exec().comm.override(
boundary_reduction=get_spec().speculative_boundary_reduction
):
yield
finally:
moe.in_speculative_scope = original_scope
moe.disable_shared_experts_fusion = original_fusion
@@ -759,10 +764,8 @@ def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool:
Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one.
"""
return (
should_skip_mlp_all_reduce()
or should_use_dp_reduce_scatterv()
or post_experts_output_is_complete(is_tp_path=is_tp_path)
return should_skip_mlp_all_reduce() or post_experts_output_is_complete(
is_tp_path=is_tp_path
)
@@ -565,7 +565,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_prefill_cuda_graph_max_prefix_len: int = 0
global_forward_mode: Optional[ForwardMode] = None
# Owned only while this invocation executes its local decoder stack.
# Current layer-stack invocation; each TBO child owns a separate stream.
residual_stream: Optional[ResidualStream] = None
# For two-batch overlap
+41 -58
View File
@@ -36,9 +36,10 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateList,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -673,15 +674,6 @@ class BailingMoEBlock(nn.Module):
config, layer_id=layer_id + 1, is_nextn=False
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# A NextN draft is a one-layer model.
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = BailingMoESparseMoeBlock(
layer_id=layer_id,
@@ -706,11 +698,21 @@ class BailingMoEBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
@@ -725,18 +727,14 @@ class BailingMoEBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
capture_output=None,
) -> torch.Tensor:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
)
hidden_states = self.attn_stage.prepare(
hidden_states,
forward_batch,
captured_last_layer_outputs=captured_last_layer_outputs,
capture_output=capture_output,
)
if hidden_states.shape[0] != 0:
@@ -746,17 +744,17 @@ class BailingMoEBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class BailingMoEModel(nn.Module):
@@ -821,51 +819,36 @@ class BailingMoEModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
captured_last_layer_outputs=(
aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None
),
captured_last_layer_outputs=aux_hidden_states
if getattr(layer, "_is_layer_to_capture", False)
else None,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
+37 -48
View File
@@ -16,8 +16,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -814,26 +815,28 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# A NextN draft is a one-layer model.
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=is_moe_layer,
is_previous_layer_sparse=is_previous_moe_layer,
is_next_layer_sparse=is_next_layer_moe_layer,
)
qkv_latent_func = (
self.attention.prepare_qkv_latent
if config.attention_type == 1 and self.use_mla
else None
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=False,
qkv_latent_func=qkv_latent_func,
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{"qkv_latent_func": qkv_latent_func},
),
(
declare_ffn(
sparse=is_moe_layer,
next_sparse=is_next_layer_moe_layer,
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
def _is_layer_sparse(
@@ -848,13 +851,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}"
# )
@@ -875,16 +875,15 @@ class BailingMoELinearDecoderLayer(nn.Module):
# logger.warning(
# f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}"
# )
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
# logger.warning(
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
# )
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
@staticmethod
def shared_moe_coefficient_loader(
@@ -981,12 +980,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = inputs_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
total_num_layers = self.end_layer - self.start_layer
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
@@ -999,30 +998,20 @@ class BailingMoELinearModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
+7 -13
View File
@@ -162,7 +162,7 @@ class BailingMoEModelNextN(nn.Module):
)
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
if self.is_hybrid:
device = input_ids.device
zero_allocator = BumpAllocator(
@@ -172,26 +172,20 @@ class BailingMoEModelNextN(nn.Module):
dtype=torch.float32,
device=device,
)
hidden_states, residual = self.decoder(
hidden_states = self.decoder(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
)
else:
hidden_states, residual = self.decoder(
positions, hidden_states, forward_batch, residual
)
hidden_states = self.decoder(positions, hidden_states, forward_batch)
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is not None:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
else:
hidden_states = self.final_layernorm(hidden_states)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.final_layernorm
)
return hidden_states
+44 -61
View File
@@ -30,9 +30,10 @@ from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1148,27 +1149,27 @@ class BailingMoELinearDecoderLayer(nn.Module):
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
# NextN wraps a single decoder layer whose checkpoint prefix is the
# post-model layer id. Treat it as a one-layer model for scatter-mode
# planning so A2A/DeepEP outputs are gathered before logits.
num_layers=1 if is_nextn else config.num_hidden_layers,
is_layer_sparse=is_moe_layer,
is_previous_layer_sparse=is_previous_moe_layer,
is_next_layer_sparse=is_next_layer_sparse,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=(
self.attention.prepare_qkv_latent
if self.attention_type == 1 and self.use_mla
else None
self.attn_stage, self.ffn_stage = make_stages(
(
declare_attn(),
self.input_layernorm,
{
"qkv_latent_func": self.attention.prepare_qkv_latent
if self.attention_type == 1 and self.use_mla
else None
},
),
(
declare_ffn(
sparse=is_moe_layer,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
if layer_id != 0
else None,
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
)
@torch.inference_mode()
@@ -1177,18 +1178,12 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
zero_allocator: BumpAllocator,
capture_output=None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
if not forward_batch.forward_mode.is_idle():
@@ -1213,11 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if not (
enable_moe_dense_fully_dp()
and (not self.is_layer_sparse)
@@ -1227,9 +1221,9 @@ class BailingMoELinearDecoderLayer(nn.Module):
hidden_states,
forward_batch=forward_batch,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
@staticmethod
def shared_moe_coefficient_loader(
@@ -1347,12 +1341,12 @@ class BailingMoELinearModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
total_num_layers = self.end_layer - self.start_layer
device = hidden_states.device
@@ -1386,41 +1380,30 @@ class BailingMoELinearModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
residual=residual,
zero_allocator=zero_allocator,
capture_output=capture_output
if capture_aux
and i > self.start_layer
and i - 1 in self.layers_to_capture
and (i - 1 in self.layers_to_capture)
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states = residual_batch.norm(
hidden_states,
residual,
forward_batch,
self.norm,
capture_output=(
capture_output
if capture_aux and self.end_layer - 1 in self.layers_to_capture
else None
),
capture_output=capture_output
if capture_aux and self.end_layer - 1 in self.layers_to_capture
else None,
)
if capture_aux and len(dspark_aux_hidden_states) > 0:
return hidden_states, dspark_aux_hidden_states
+6 -1
View File
@@ -3794,7 +3794,12 @@ class DeepseekV4DecoderLayer(nn.Module):
and forward_batch.dp_padding_mode.is_max_len()
and get_parallel().tp_size == get_parallel().attn_dp_size
)
mlp_reduce_scatter = _use_cp or _use_reduce_scatterv or _use_reduce_scatter
mlp_reduce_scatter = (
_use_cp
or _use_reduce_scatterv
or _use_reduce_scatter
or (_use_tp_moe_gather and should_use_dp_reduce_scatterv())
)
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): compute the replicated shared expert
# on LOCAL hidden before the gather and add it back after the combine
# (reduce_scatterv OR dp_scatter), instead of on the gathered global buffer.
+38 -35
View File
@@ -11,7 +11,12 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.output import OutputTransform
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -90,9 +95,11 @@ class FalconH1MLP(nn.Module):
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
x = x * self.down_multiplier
return x
def scale_output(self, x):
return x * self.down_multiplier
class FalconH1HybridAttentionDecoderLayer(nn.Module):
def __init__(
@@ -199,14 +206,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
is_previous_layer_sparse = False
is_next_layer_sparse = False
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
self.feed_forward = FalconH1MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
@@ -223,12 +222,24 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.pre_ff_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
output=OutputTransform(
self.feed_forward.scale_output, before_reduce_scatter=True
),
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.pre_ff_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
self.alt_stream = alt_stream
@@ -320,13 +331,10 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
**kwargs: Any,
):
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
# Attention block
@@ -355,12 +363,11 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
hidden_states = attention_hidden_states + mamba_hidden_states
# Fully Connected
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
ALL_DECODER_LAYER_TYPES = {
@@ -422,26 +429,22 @@ class FalconH1Model(nn.Module):
else:
hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
layer_id=i,
positions=positions,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.final_layernorm(hidden_states)
else:
hidden_states, _ = self.final_layernorm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.final_layernorm
)
return hidden_states
+39 -57
View File
@@ -31,8 +31,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -570,14 +571,6 @@ class GptOssDecoderLayer(nn.Module):
is_previous_layer_sparse = True
is_next_layer_sparse = True
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = GptOssSparseMoeBlock(
layer_id=self.layer_id,
@@ -606,11 +599,21 @@ class GptOssDecoderLayer(nn.Module):
x_pad_to_multiple=post_attn_pad_multiple,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -618,16 +621,10 @@ class GptOssDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
capture_output=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
residual,
forward_batch,
capture_output=capture_output,
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
@@ -637,15 +634,14 @@ class GptOssDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class GptOssModel(nn.Module):
@@ -708,12 +704,12 @@ class GptOssModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
# Capture hidden-state boundaries: boundary 0 is the embedding output,
# and boundary i + 1 is the output after transformer block i.
@@ -721,44 +717,30 @@ class GptOssModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = last_layer.layer_communicator.norm_output(
hidden_states = residual_batch.norm(
hidden_states,
residual,
forward_batch,
self.norm,
capture_output=(
aux_hidden_states.append
if self.end_layer in self.layers_to_capture
else None
),
capture_output=aux_hidden_states.capture
if self.end_layer in self.layers_to_capture
else None,
)
elif self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
residual_batch.snapshot(hidden_states, forward_batch)
)
if len(aux_hidden_states) == 0:
return hidden_states
+35 -54
View File
@@ -32,9 +32,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
enable_moe_dense_fully_dp,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -613,14 +614,6 @@ class LLaDA2MoeBlock(nn.Module):
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=is_previous_layer_sparse,
is_next_layer_sparse=is_next_layer_sparse,
)
if self.is_layer_sparse:
self.mlp = LLaDA2MoeSparseMoeBlock(
layer_id=layer_id,
@@ -645,12 +638,21 @@ class LLaDA2MoeBlock(nn.Module):
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
@@ -663,12 +665,9 @@ class LLaDA2MoeBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
hidden_states = self.attn_stage.prepare(
hidden_states=hidden_states, forward_batch=forward_batch
)
hidden_states = self.attention(
@@ -677,17 +676,17 @@ class LLaDA2MoeBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class LLaDA2MoeModel(nn.Module):
@@ -747,44 +746,26 @@ class LLaDA2MoeModel(nn.Module):
hidden_states = self.word_embeddings(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
+41 -52
View File
@@ -25,7 +25,12 @@ import torch
from torch import nn
from transformers import Llama4TextConfig
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -421,20 +426,19 @@ class Llama4DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=is_moe_layer,
is_previous_layer_sparse=is_previous_moe_layer,
is_next_layer_sparse=is_next_moe_layer,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=is_moe_layer,
next_sparse=is_next_moe_layer,
),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _is_moe_layer(self, layer_id: int) -> bool:
@@ -453,10 +457,10 @@ class Llama4DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
capture_output=None,
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(
hidden_states, forward_batch, capture_output=capture_output
)
if hidden_states.shape[0] != 0:
@@ -466,16 +470,15 @@ class Llama4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
# Fully Connected
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class Llama4Model(nn.Module):
@@ -519,28 +522,22 @@ class Llama4Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
aux_hidden_states = []
residual_batch.start(forward_batch)
aux_hidden_states = AuxHiddenStateList()
for i in range(len(self.layers)):
if i in self.layers_to_capture:
aux_hidden_states.append(
self.layers[i].layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -568,29 +565,21 @@ class Llama4ForCausalLM(LlamaForCausalLM):
):
start, end = split_interval
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
forward_batch.hidden_states = (
self.model.embed_tokens(input_ids)
if input_embeds is None
else input_embeds
)
for layer in self.model.layers[start:end]:
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
forward_batch.hidden_states = layer(
positions, forward_batch.hidden_states, forward_batch
)
if end != self.model.config.num_hidden_layers:
return None
hidden, residual = LayerCommunicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.residual = None
if residual is None:
forward_batch.hidden_states = self.model.norm(hidden)
else:
forward_batch.hidden_states, _ = self.model.norm(hidden, residual)
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
)
+28 -46
View File
@@ -77,7 +77,7 @@ from sglang.srt.model_loader.weight_utils import (
replace_prefix,
replace_substrings,
)
from sglang.srt.models.nemotron_h_utils import make_layer_communicator
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
from sglang.srt.utils import (
@@ -375,8 +375,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
mixer's attention partial sum, or a value that is complete or carries the
sum an FFN before it left."""
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
self.layer_communicator = make_layer_communicator(
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
self.stage = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -384,15 +384,12 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
self,
*,
hidden_states,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
):
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.stage.prepare(hidden_states, forward_batch)
with self.stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mixer.forward(hidden_states)
return ffn_exit.finish(hidden_states, residual)
return ffn_exit.finish(hidden_states)
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -426,7 +423,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx)
self._init_stage_boundary(config, layer_idx)
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -449,7 +446,7 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx)
self._init_stage_boundary(config, layer_idx)
class NemotronHAttnLikeDecoderLayer(nn.Module):
@@ -457,8 +454,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
output all-reduce to that stage's input; otherwise it reduces the output
itself unless the fused kernel takes the sum into the next input norm."""
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
self.layer_communicator = make_layer_communicator(
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
self.stage = make_stage_boundary(
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
)
@@ -466,20 +463,17 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
self,
*,
hidden_states,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
):
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.stage.prepare(hidden_states, forward_batch)
if forward_batch.forward_mode.is_idle():
return hidden_states, residual
return hidden_states
with self.layer_communicator.mixer_exit(forward_batch) as mixer_exit:
with self.stage.exit(forward_batch) as mixer_exit:
hidden_states = self._forward_mixer(
hidden_states, forward_batch, mixer_exit.skips_reduction
)
return mixer_exit.finish(hidden_states, residual)
return mixer_exit.finish(hidden_states)
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
@@ -507,7 +501,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx)
self._init_stage_boundary(config, layer_idx)
def _forward_mamba(
self,
@@ -645,7 +639,7 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_layer_communicator(config, layer_idx)
self._init_stage_boundary(config, layer_idx)
def _forward_mixer(
self,
@@ -733,50 +727,38 @@ class NemotronHModel(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states, snapshot = self.layers[
i
].layer_communicator.capture_output(
hidden_states, residual, at_input=True
hidden_states, snapshot = self.layers[i].stage.capture_output(
hidden_states, forward_batch, at_input=True
)
aux_hidden_states.append(snapshot)
layer = self.layers[i]
if not isinstance(layer, Layers):
raise ValueError(f"Unknown layer type: {type(layer)}")
hidden_states, residual = layer.forward(
hidden_states = layer.forward(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
return residual_batch.to_pp(
hidden_states, forward_batch, preserve_declared=True
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if self.end_layer in self.layers_to_capture:
aux_hidden_states.append(
last_layer.layer_communicator.snapshot(hidden_states, residual)
residual_batch.snapshot(hidden_states, forward_batch)
)
if residual is None:
hidden_states = self.norm_f(hidden_states)
else:
hidden_states, _ = self.norm_f(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm_f)
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
+23 -30
View File
@@ -89,9 +89,8 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
*,
inputs_embeds: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
if self.has_start_projections:
inputs_embeds_normed = self.enorm(inputs_embeds)
previous_hidden_states_normed = self.hnorm(hidden_states)
@@ -105,23 +104,22 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
hidden_states, dim=-1
)
hidden_states, residual = super().forward(
if self.has_start_projections:
hidden_states = residual_batch.written(hidden_states, forward_batch)
hidden_states = super().forward(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
if self.has_end_norm:
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states, residual = self.layer_communicator.fold(
hidden_states, residual
hidden_states = residual_batch.fold(hidden_states, forward_batch)
hidden_states = residual_batch.written(
self.final_layernorm(hidden_states), forward_batch
)
hidden_states = self.final_layernorm(hidden_states)
return hidden_states, residual
return hidden_states
class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
@@ -173,9 +171,8 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
*,
inputs_embeds: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
if self.has_start_projections:
inputs_embeds_normed = self.enorm(inputs_embeds)
previous_hidden_states_normed = self.hnorm(hidden_states)
@@ -189,23 +186,22 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
hidden_states, dim=-1
)
hidden_states, residual = super().forward(
if self.has_start_projections:
hidden_states = residual_batch.written(hidden_states, forward_batch)
hidden_states = super().forward(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
if self.has_end_norm:
hidden_states, residual = self.layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states, residual = self.layer_communicator.fold(
hidden_states, residual
hidden_states = residual_batch.fold(hidden_states, forward_batch)
hidden_states = residual_batch.written(
self.final_layernorm(hidden_states), forward_batch
)
hidden_states = self.final_layernorm(hidden_states)
return hidden_states, residual
return hidden_states
class NemotronHMultiTokenPredictor(nn.Module):
@@ -302,18 +298,15 @@ class NemotronHMultiTokenPredictor(nn.Module):
inputs_embeds = self.get_input_embeddings(input_ids)
hidden_states = forward_batch.spec_info.hidden_states
residual = None
residual_batch.start(forward_batch)
for i in range(self.pattern_len):
if residual is None:
residual = residual_batch.start(forward_batch)
hidden_states, residual = self.layers[str(i)](
hidden_states = self.layers[str(i)](
inputs_embeds=inputs_embeds,
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
return hidden_states
return residual_batch.take_output(hidden_states, forward_batch)
class NemotronHForCausalLMMTP(NemotronHForCausalLM):
+4 -4
View File
@@ -101,14 +101,12 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
)
def make_layer_communicator(
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
) -> LayerCommunicator:
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
"""The communicator of a layer that is one stage: only its own norm, and
boundaries built from the stages next to it in the pattern."""
stage = layer_stage(pattern, layer_idx)
for_attn = stage.kind is StageKind.ATTENTION
return LayerCommunicator(
boundaries = LayerCommunicator(
layer_facts=LayerFacts(
is_layer_sparse=pattern[layer_idx] == MOE,
is_last_layer=layer_idx == len(pattern) - 1,
@@ -121,3 +119,5 @@ def make_layer_communicator(
allow_reduce_scatter=not for_attn,
stage=stage,
)
return boundaries.attn if for_attn else boundaries.ffn
+53 -83
View File
@@ -1,12 +1,16 @@
# Adapted from qwen2.py
import logging
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch
from torch import nn
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator.residual import access as residual_access
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -372,18 +376,14 @@ class Qwen3DecoderLayer(nn.Module):
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
is_previous_layer_sparse=False,
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=False, next_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -391,15 +391,15 @@ class Qwen3DecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
post_residual_addition: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
capture_output=None,
) -> torch.Tensor:
# Self Attention
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states = self.attn_stage.prepare(
hidden_states,
residual,
forward_batch,
post_residual_addition=post_residual_addition,
capture_output=capture_output,
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -409,9 +409,9 @@ class Qwen3DecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(
hidden_states,
residual,
forward_batch,
cache=(
[self.mlp.gate_up_proj.weight, self.mlp.down_proj.weight]
@@ -424,12 +424,12 @@ class Qwen3DecoderLayer(nn.Module):
else None
),
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class Qwen3Model(Qwen2Model):
@@ -450,60 +450,39 @@ class Qwen3Model(Qwen2Model):
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> Union[torch.Tensor, PPProxyTensors]:
input_ids,
positions,
forward_batch,
input_embeds=None,
pp_proxy_tensors=None,
):
if self.pp_group.is_first_rank:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
aux_hidden_states = []
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(
residual_access.snapshot(hidden_states, residual)
)
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = self.layers[i](
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
hidden_states, residual = residual_access.finish_layer_stack(
hidden_states, residual, forward_batch
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) == 0:
return hidden_states
return hidden_states, aux_hidden_states
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return (
(hidden_states, aux_hidden_states) if aux_hidden_states else hidden_states
)
class Qwen3ForCausalLM(nn.Module):
@@ -613,7 +592,7 @@ class Qwen3ForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
forward_batch.residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -621,25 +600,16 @@ class Qwen3ForCausalLM(nn.Module):
# decoder layer
for i in range(start, end):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
forward_batch.hidden_states = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
+27 -39
View File
@@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import (
@@ -1205,24 +1206,23 @@ class Qwen3LLMModel(Qwen3Model):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for layer_idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer]
):
layer_idx = layer_idx + self.start_layer
if layer_idx in self.layers_to_capture:
aux_hidden_states.append(
layer.layer_communicator.snapshot(
hidden_states, residual, at_input=True
)
)
capture_output = (
aux_hidden_states.capture
if layer_idx in self.layers_to_capture
else None
)
if self.use_hf_deepstack_order:
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
@@ -1231,40 +1231,33 @@ class Qwen3LLMModel(Qwen3Model):
deepstack_embeds = self.get_deepstack_embeds(
layer_idx - 1, input_deepstack_embeds
)
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
post_residual_addition=deepstack_embeds,
capture_output=capture_output,
)
else:
# Inference path: add deepstack directly to hidden_states at the end of the layer
# (original, grounding-correct order).
hidden_states, residual = layer(
hidden_states = layer(
positions,
hidden_states,
forward_batch,
residual,
capture_output=capture_output,
)
if (
input_deepstack_embeds is not None
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
hidden_states.add_(
input_deepstack_embeds[:, sep : sep + self.hidden_size]
hidden_states = residual_batch.add_to_output(
hidden_states,
forward_batch,
input_deepstack_embeds[:, sep : sep + self.hidden_size],
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
# Handle deepstack for the last processed layer (HF-order path only).
last_deepstack = (
self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds)
@@ -1273,20 +1266,15 @@ class Qwen3LLMModel(Qwen3Model):
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = residual_batch.norm(
hidden_states,
forward_batch,
self.norm,
post_residual_addition=last_deepstack,
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(
hidden_states, residual, post_residual_addition=last_deepstack
)
if len(aux_hidden_states) == 0:
return hidden_states
+5 -3
View File
@@ -72,7 +72,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
make_ple_file_prefetcher,
make_ple_file_rss_trimmer,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, logger
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -1439,14 +1439,16 @@ class Qwen4ExpLayerExtensionMixin:
attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size))
hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous()
hidden_states = self.mlp(hidden_states, forward_batch)
use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv()
with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv):
hidden_states = self.mlp(hidden_states, forward_batch)
if use_dp_moe_gather:
hidden_states, global_hidden_states = (
get_local_dp_buffer(get_parallel().tp_group),
hidden_states,
)
if should_use_dp_reduce_scatterv():
if use_reduce_scatterv:
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
+8 -18
View File
@@ -1456,36 +1456,26 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
start, end = split_interval
if start == 0:
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.word_embeddings(input_ids)
else:
forward_batch.hidden_states = input_embeds
forward_batch.residual = residual_batch.start(forward_batch)
for i in range(start, end):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
forward_batch.hidden_states = layer(
positions, forward_batch.hidden_states, forward_batch
)
if end == self.model.config.num_hidden_layers:
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
end - 1
].layer_communicator.finish_layer_stack(
forward_batch.hidden_states, forward_batch.residual, forward_batch
forward_batch.hidden_states = residual_batch.finish(
forward_batch.hidden_states, forward_batch
)
forward_batch.hidden_states = residual_batch.norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
if forward_batch.residual is None:
hidden_states = self.model.norm(forward_batch.hidden_states)
else:
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
)
forward_batch.residual = None
forward_batch.hidden_states = hidden_states
return self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
+31 -49
View File
@@ -11,7 +11,11 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -291,19 +295,17 @@ class SDARBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=False,
is_previous_layer_sparse=False,
is_next_layer_sparse=False,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=False,
next_sparse=False,
),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -311,13 +313,8 @@ class SDARBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
residual,
forward_batch,
)
) -> torch.Tensor:
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -326,17 +323,14 @@ class SDARBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states,
residual,
forward_batch,
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class SDARModel(nn.Module):
@@ -404,37 +398,25 @@ class SDARModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch, allow_missing_residual=True
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states, residual = self.layers[
self.end_layer - 1
].layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
hidden_states = layer(positions, hidden_states, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
else:
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states, residual = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
+28 -44
View File
@@ -14,8 +14,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerFacts,
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.dp_attention import (
@@ -356,19 +357,16 @@ class SDARMoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=True,
is_previous_layer_sparse=True,
is_next_layer_sparse=True,
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(sparse=True, next_sparse=True),
self.post_attention_layernorm,
),
previous=declare_ffn(sparse=True, next_sparse=True)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def forward(
@@ -376,12 +374,9 @@ class SDARMoeBlock(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -390,18 +385,17 @@ class SDARMoeBlock(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
with self.ffn_stage.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(
hidden_states,
forward_batch=forward_batch,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class SDARMoeModel(nn.Module):
@@ -470,34 +464,24 @@ class SDARMoeModel(nn.Module):
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = self.layers[
self.start_layer
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
pp_proxy_tensors, forward_batch
)
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
with get_global_expert_distribution_recorder().with_current_layer(i):
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states = layer(positions, hidden_states, forward_batch)
last_layer = self.layers[self.end_layer - 1]
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
hidden_states,
residual,
forward_batch,
preserve_declared=not self.pp_group.is_last_rank,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if not forward_batch.forward_mode.is_idle():
hidden_states, residual = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
+36 -42
View File
@@ -16,13 +16,14 @@ from sglang.srt.configs.step3_vl import (
Step3VisionEncoderConfig,
Step3VLConfig,
)
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
from sglang.srt.layers.communicator import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual import batch as residual_batch
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
@@ -40,6 +41,7 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import reduce_moe_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -78,6 +80,7 @@ class Step3TextMLP(nn.Module):
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -93,6 +96,7 @@ class Step3TextMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
reduce_results=reduce_results,
)
if hidden_act != "silu":
raise ValueError(
@@ -163,8 +167,6 @@ class Step3TextMoEMLP(nn.Module):
hidden_states=hidden_states, topk_output=topk_output
)
if self.tp_size > 1:
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
@@ -339,14 +341,6 @@ class Step3TextDecoderLayer(nn.Module):
)
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
self.layer_facts = LayerFacts.init_new(
layer_id=layer_id,
num_layers=config.num_hidden_layers,
is_layer_sparse=self.is_layer_sparse,
is_previous_layer_sparse=self.is_previous_layer_sparse,
is_next_layer_sparse=self.is_next_layer_sparse,
)
if not self.is_layer_sparse:
self.mlp = Step3TextMLP(
hidden_size=config.hidden_size,
@@ -370,6 +364,7 @@ class Step3TextDecoderLayer(nn.Module):
hidden_act="silu",
quant_config=quant_config,
prefix=add_prefix("share_expert", prefix),
reduce_results=False,
)
else:
self.moe = Step3TextMoEMLP(
@@ -379,11 +374,21 @@ class Step3TextDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
self.layer_communicator = LayerCommunicator(
layer_facts=self.layer_facts,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
self.attn_stage, self.ffn_stage = make_stages(
(declare_attn(), self.input_layernorm),
(
declare_ffn(
sparse=self.is_layer_sparse,
next_sparse=self.is_next_layer_sparse,
),
self.post_attention_layernorm,
),
previous=declare_ffn(
sparse=self.is_previous_layer_sparse, next_sparse=self.is_layer_sparse
)
if layer_id != 0
else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def moe_mlp_forward(self, hidden_states):
@@ -393,19 +398,16 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states += self.share_expert(h)
else:
hidden_states = self.moe(hidden_states)
return hidden_states
return reduce_moe_output(hidden_states)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
@@ -414,17 +416,16 @@ class Step3TextDecoderLayer(nn.Module):
forward_batch=forward_batch,
)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
with self.ffn_stage.exit(forward_batch) as ffn_exit:
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states, residual
return hidden_states
class Step3TextModel(nn.Module):
@@ -472,22 +473,15 @@ class Step3TextModel(nn.Module):
else:
hidden_states = input_embeds
residual = residual_batch.start(forward_batch)
residual_batch.start(forward_batch)
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
hidden_states, residual, forward_batch
)
hidden_states = residual_batch.finish(hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
return hidden_states
+125
View File
@@ -0,0 +1,125 @@
"""Small fixtures around production boundaries; no parallel layout recipes."""
from types import SimpleNamespace
from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan
from sglang.srt.layers.communicator.factories import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
from sglang.srt.layers.communicator.stage import StageCommunicator
def make_test_stages(
*,
attention_norm,
ffn_norm,
first=False,
last=False,
sparse=False,
previous_sparse=False,
next_sparse=False,
residual=PLAIN_RESIDUAL,
output=None,
**options,
):
previous = (
None
if first
else declare_ffn(
sparse=previous_sparse, next_sparse=sparse, update=residual.ffn_update
)
)
attention = declare_attn(
read=residual.attention_read, update=residual.attention_update
)
ffn = declare_ffn(
sparse=sparse,
next_sparse=next_sparse,
read=residual.ffn_read,
update=residual.ffn_update,
output=output,
)
common = {
key: options.pop(key)
for key in ("force_layernorm_before_dp_gather", "fusions")
if key in options
}
attn, ffn = make_stages(
(
attention,
attention_norm,
dict(
residual_in_hidden=residual.ffn_update.at_producer, **common, **options
),
),
(ffn, ffn_norm, common),
previous=previous,
terminal=last,
)
return SimpleNamespace(attn=attn, ffn=ffn)
def stub_plan():
"""Supply only defaults for tests that inject preselected paths/kernels."""
plan = StagePlan.__new__(StagePlan)
plan.norm = None
plan.fusions = None
plan._publish_lora_layout = False
plan._fusion_rows = None
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
plan.is_first_layer = False
plan.residual_in_hidden = False
# Keep production variant selection while tests replace individual paths.
plan._batch_steps = lambda batch: {
BatchVariant.ORDINARY: plan._steps,
BatchVariant.CONTEXT_PARALLEL: plan._cp_steps,
BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps,
BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps,
}[plan._variant(batch)]
return plan
def stub_stage(plan, kind):
"""Keep the same concrete boundary when a test patches its methods."""
stages = plan.__dict__.setdefault("_stub_stages", {})
if kind not in stages:
stages[kind] = StageCommunicator(plan, kind, plan.norm)
return stages[kind]
def sp_region_steps():
"""Local SP rows with no owed sum, for tests of activation and exits."""
from sglang.srt.layers.communicator import (
NORM_QUANT_READ,
NORM_READ,
BoundarySteps,
CommunicateSummableTensorPairFn,
EdgeDecl,
Layout,
StageEntry,
StageInput,
StageOutput,
TokenAxis,
make_boundary,
)
from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
output = StageOutput(rows)
def entry(read, handoff=None):
selected = make_boundary(
EdgeDecl(output, StageInput(rows, read=read), rows, rows)
)
return StageEntry(selected.prepare, rows, handoff=handoff)
return BoundarySteps(
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input),
entry(NORM_READ),
output,
CommunicateSummableTensorPairFn._trivial,
False,
)
+35 -5
View File
@@ -8,14 +8,40 @@ the name, not only into the package.
import contextlib
from unittest import mock
from sglang.srt.layers import communicator
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
from sglang.srt.layers import communicator as communicator
from sglang.srt.layers.communicator import (
boundary,
construction,
exit,
factories,
layer,
layout,
legacy_stage,
ops,
output,
)
from sglang.srt.layers.communicator import residual as residual_contract
from sglang.srt.layers.communicator.adapters import attention
from sglang.srt.layers.communicator.residual import access, add_norm, mhc
from sglang.srt.layers.communicator import (
stage,
)
from sglang.srt.layers.communicator.adapters import attention, branch, lora
from sglang.srt.layers.communicator.fusions import allreduce
from sglang.srt.layers.communicator.residual import (
access,
add_norm,
batch,
mhc,
stream,
)
COMMUNICATOR_MODULES = (
layer,
legacy_stage,
communicator,
allreduce,
exit,
branch,
lora,
layout,
output,
attention,
@@ -25,7 +51,11 @@ COMMUNICATOR_MODULES = (
mhc,
ops,
boundary,
layer,
construction,
factories,
stage,
batch,
stream,
)