[Refactor] Construct independent decoder stage boundaries (#41552)
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""Resolve collective preferences before constructing target and draft models."""
|
||||
|
||||
import json
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import model_config_of
|
||||
|
||||
_RSV_ARCHITECTURES = {
|
||||
"BailingMoELinearForCausalLM",
|
||||
"BailingMoeV2_5ForCausalLM",
|
||||
"LongcatFlashForCausalLM",
|
||||
"LongcatFlashForCausalLMNextN",
|
||||
"Step3VLForConditionalGeneration",
|
||||
}
|
||||
_AR_ARCHITECTURES = {
|
||||
"Qwen3ForCausalLM",
|
||||
"Qwen3ForSequenceClassification",
|
||||
"Qwen3Model",
|
||||
"Qwen3ForRewardModel",
|
||||
"MossVLForCausalLM",
|
||||
"MossVLForConditionalGeneration",
|
||||
}
|
||||
|
||||
|
||||
def model_boundary_reduction(config):
|
||||
"""One default for a model's dense and MoE layers, including its MTP."""
|
||||
architectures = set(getattr(config, "architectures", None) or ())
|
||||
if architectures & _RSV_ARCHITECTURES:
|
||||
return "rsv"
|
||||
if architectures & _AR_ARCHITECTURES:
|
||||
return "ar"
|
||||
# MiniCPM-V 4.5 constructs Qwen3 from its top-level config.
|
||||
if (
|
||||
getattr(config, "model_type", None) == "minicpmv"
|
||||
and str(getattr(config, "version", "")) == "4.5"
|
||||
):
|
||||
return "ar"
|
||||
# Wrappers carry their actual language backbone in the nested HF config.
|
||||
# Match Qwen3 specifically: Qwen3-MoE/Next/3.5 fall through to rs+rsv.
|
||||
if getattr(config, "model_type", None) in ("qwen3", "qwen3_vl_text"):
|
||||
return "ar"
|
||||
get_text = getattr(config, "get_text_config", None)
|
||||
if get_text is not None:
|
||||
text = get_text()
|
||||
if text is not config:
|
||||
return model_boundary_reduction(text)
|
||||
for name in ("text_config", "llm_config", "language_config"):
|
||||
text = getattr(config, name, None)
|
||||
if text is not None and text is not config:
|
||||
return model_boundary_reduction(text)
|
||||
return "rs+rsv"
|
||||
|
||||
|
||||
def resolve_boundary_reduction(view):
|
||||
"""Resolve auto once, after speculative paths/revisions have been resolved."""
|
||||
requested = view.boundary_reduction
|
||||
dummy = view.model_path.lower() in ("none", "dummy")
|
||||
if requested != "auto":
|
||||
target = draft = requested
|
||||
else:
|
||||
target = draft = (
|
||||
"rs+rsv"
|
||||
if dummy
|
||||
else model_boundary_reduction(model_config_of(view).hf_config)
|
||||
)
|
||||
draft_path = view.speculative_draft_model_path
|
||||
if (
|
||||
not dummy
|
||||
and view.speculative_algorithm is not None
|
||||
and draft_path is not None
|
||||
and draft_path != view.model_path
|
||||
):
|
||||
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||
|
||||
kwargs = {}
|
||||
if view.decrypted_draft_config_file:
|
||||
kwargs["_configuration_file"] = view.decrypted_draft_config_file
|
||||
config = get_config(
|
||||
draft_path,
|
||||
trust_remote_code=view.trust_remote_code,
|
||||
revision=view.speculative_draft_model_revision,
|
||||
model_override_args=json.loads(view.json_model_override_args),
|
||||
**kwargs,
|
||||
)
|
||||
draft = model_boundary_reduction(config)
|
||||
return {
|
||||
"boundary_reduction": target,
|
||||
"speculative_boundary_reduction": draft,
|
||||
}
|
||||
@@ -250,6 +250,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"enable_torch_symm_mem",
|
||||
"enable_scattered_sconv",
|
||||
"pre_warm_nccl",
|
||||
"boundary_reduction",
|
||||
"enable_quant_communications",
|
||||
"enable_flashinfer_allreduce_fusion",
|
||||
"enforce_disable_flashinfer_allreduce_fusion",
|
||||
|
||||
@@ -620,6 +620,19 @@ class ExecComm(msgspec.Struct):
|
||||
bool,
|
||||
"Pre-warm NCCL/RCCL communicators during startup to reduce P99 TTFT cold-start latency. Default: enabled for AMD/HIP (RCCL), disabled for NVIDIA/CUDA (NCCL).",
|
||||
] = False
|
||||
boundary_reduction: A[
|
||||
Literal["auto", "ar", "rs", "rsv", "rs+rsv"],
|
||||
Arg(
|
||||
help="Select FFN boundary reduction: ar uses all-reduce then token "
|
||||
"redistribution; rs and rsv permit fixed-size and variable-size "
|
||||
"reduce-scatter respectively; rs+rsv permits both (RSv first). "
|
||||
"Unsupported paths fall back to ar. The option applies only to FFN "
|
||||
"layers using stage boundaries, where auto resolves the model default. "
|
||||
"Required attention and MoE collectives and "
|
||||
"all-reduce fusion are unaffected.",
|
||||
resolvable=True,
|
||||
),
|
||||
] = "auto"
|
||||
enable_quant_communications: A[
|
||||
Optional[bool],
|
||||
"Enable INT8 quantization of TP communications (limited support).",
|
||||
|
||||
@@ -39,6 +39,10 @@ class Spec(msgspec.Struct):
|
||||
aliases=["--speculative-draft-model"],
|
||||
),
|
||||
] = None
|
||||
speculative_boundary_reduction: A[
|
||||
Literal["ar", "rs", "rsv", "rs+rsv"],
|
||||
Arg(no_cli=True, resolvable=True),
|
||||
] = "rs+rsv"
|
||||
speculative_draft_model_revision: A[
|
||||
Optional[str],
|
||||
"The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.",
|
||||
|
||||
@@ -92,6 +92,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
run_hook(validate_prefill_cp_platform, server_args)
|
||||
run_hook(handle_hardware_runtime_validation, server_args)
|
||||
if cfg.model_path.lower() in ["none", "dummy"]:
|
||||
from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
|
||||
|
||||
run_post_process_pass(server_args, resolve_boundary_reduction)
|
||||
return
|
||||
|
||||
from sglang.srt.arg_groups.model_path_hook import (
|
||||
@@ -284,6 +287,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_speculative_decoding, server_args)
|
||||
|
||||
from sglang.srt.arg_groups.boundary_reduction import resolve_boundary_reduction
|
||||
|
||||
run_post_process_pass(server_args, resolve_boundary_reduction)
|
||||
|
||||
# After the speculative hook so speculative_algorithm is final.
|
||||
from sglang.srt.arg_groups.layernorm_sp_hook import handle_layernorm_sp
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from sglang.srt.batch_overlap.operations import (
|
||||
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
tbo_split_moves,
|
||||
)
|
||||
@@ -1022,14 +1021,12 @@ def _model_forward_tbo_split_inputs(
|
||||
zero_allocator: Optional[BumpAllocator],
|
||||
layer_input_rows: Layout,
|
||||
) -> List[Dict]:
|
||||
context = CommunicateContext.init_new()
|
||||
to_splitter, to_layer_input = tbo_split_moves(layer_input_rows)
|
||||
|
||||
hidden_states, residual = to_splitter(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=context,
|
||||
)
|
||||
|
||||
inputs_arr = _model_forward_tbo_split_inputs_raw(
|
||||
@@ -1045,7 +1042,6 @@ def _model_forward_tbo_split_inputs(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=context,
|
||||
)
|
||||
return dict(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Communication between a decoder layer's stages."""
|
||||
"""Stage boundaries: layout transport, residual updates, reads, and fusion."""
|
||||
|
||||
from sglang.srt.layers.communicator.adapters.attention import (
|
||||
AttentionInputs,
|
||||
@@ -19,7 +19,6 @@ from sglang.srt.layers.communicator.adapters.attention import (
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
DecoderLayerSides,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
LayerStage,
|
||||
@@ -28,25 +27,25 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
decoder_layer_edges,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
sequence_parallel_layer_sides,
|
||||
stage_edges,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layer import (
|
||||
FfnCompletion,
|
||||
FfnExit,
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
MixerExit,
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
BatchVariant,
|
||||
StageEdges,
|
||||
make_stage,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import FfnCompletion, FfnExit, MixerExit
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_attn_stage,
|
||||
make_ffn_stage,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -79,25 +78,28 @@ from sglang.srt.layers.communicator.residual.mhc import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LayerCommunicator",
|
||||
"MHCLayerCommunicator",
|
||||
"LayerFacts",
|
||||
"declare_attn",
|
||||
"declare_ffn",
|
||||
"make_attn_stage",
|
||||
"make_ffn_stage",
|
||||
"make_stages",
|
||||
"ADD",
|
||||
"AttentionInputs",
|
||||
"BoundarySteps",
|
||||
"CommunicateContext",
|
||||
"CommunicateSimpleFn",
|
||||
"CommunicateSummableTensorPairFn",
|
||||
"DecoderLayerSides",
|
||||
"EdgeDecl",
|
||||
"FUSE_ALLREDUCE_MAX_BATCH_SIZE",
|
||||
"FfnCompletion",
|
||||
"FfnExit",
|
||||
"FusedMlpInput",
|
||||
"HandoffOutput",
|
||||
"LayerCommunicator",
|
||||
"LayerResidual",
|
||||
"LayerFacts",
|
||||
"LayerStage",
|
||||
"Layout",
|
||||
"MHCLayerCommunicator",
|
||||
"MHCState",
|
||||
"MixerExit",
|
||||
"NORM_QUANT_READ",
|
||||
@@ -111,20 +113,22 @@ __all__ = [
|
||||
"SumGroup",
|
||||
"TokenAxis",
|
||||
"UnreducedOutput",
|
||||
"decoder_layer_edges",
|
||||
"decoder_layer_sides",
|
||||
"enable_moe_dense_fully_dp",
|
||||
"get_attn_tp_context",
|
||||
"input_scattered_layer_sides",
|
||||
"make_boundary",
|
||||
"make_output_boundary",
|
||||
"moe_cp_gathers_sparse_moe_input",
|
||||
"move_rows",
|
||||
"reduce_output",
|
||||
"sequence_parallel_layer_sides",
|
||||
"sparse_moe_gathers_over_moe_cp",
|
||||
"stage_edges",
|
||||
"tbo_split_moves",
|
||||
"token_axis_sizes",
|
||||
"tp_reduce_scatter",
|
||||
]
|
||||
|
||||
from sglang.srt.layers.communicator.layer import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
MHCLayerCommunicator,
|
||||
)
|
||||
|
||||
@@ -21,7 +21,6 @@ from typing import Callable
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -182,7 +181,6 @@ def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return gathered
|
||||
|
||||
|
||||
def _redistribute_to_attn_tp_shards(
|
||||
tensor: torch.Tensor, context: CommunicateContext
|
||||
) -> torch.Tensor:
|
||||
return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
def _redistribute_to_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
||||
parallel = get_parallel()
|
||||
return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank]
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Move prepared branch inputs and merge complete branch contributions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.ops import move_rows
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
|
||||
def branch_input(
|
||||
self,
|
||||
source: object,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The FFN input and residual that ``source``'s boundary read for its
|
||||
own FFN, for this layer's FFN, which branches from the same input: moved
|
||||
to the rows this FFN needs and its residual's rows."""
|
||||
rows, residual_rows, _ = source._branch_rows(forward_batch)
|
||||
to, residual_to, _ = self._branch_rows(forward_batch)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
raise RuntimeError("a branch must start from a prepared stage input")
|
||||
residual = stream.residual
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
|
||||
return (
|
||||
hidden_states,
|
||||
ResidualStream(residual) if stream is not None else residual,
|
||||
)
|
||||
|
||||
|
||||
def branch_output(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
"""This layer's complete FFN output as a branch's contribution, which
|
||||
adds to the layer's output without writing the residual: moved to the
|
||||
rows the layer hands on."""
|
||||
rows, _, to = self._branch_rows(forward_batch)
|
||||
return move_rows(hidden_states, rows, to, forward_batch)
|
||||
|
||||
|
||||
def merge_branch(
|
||||
self,
|
||||
contribution: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
source: object,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""A contribution from ``branch_output`` summed with what ``source``'s
|
||||
layer hands on, ``hidden_states`` and ``residual``, moved to the rows
|
||||
this layer hands on."""
|
||||
_, _, rows = source._branch_rows(forward_batch)
|
||||
_, _, to = self._branch_rows(forward_batch)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
update = None
|
||||
if stream is not None:
|
||||
stream.check(hidden_states)
|
||||
update = stream.pending.update
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, rows, to, forward_batch)
|
||||
output = contribution + hidden_states
|
||||
if stream is not None:
|
||||
stream.write(residual)
|
||||
return stream.leave(output, update), stream
|
||||
return output, residual
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Publish the token rows consumed by the selected stage to LoRA kernels."""
|
||||
|
||||
from sglang.srt.layers.communicator.layout import TokenAxis
|
||||
from sglang.srt.runtime_context import LoRABatchLayout, get_forward
|
||||
|
||||
|
||||
def publish_attention(enabled):
|
||||
if enabled:
|
||||
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
|
||||
|
||||
|
||||
def publish_ffn(enabled, input_rows):
|
||||
if enabled:
|
||||
get_forward().set(
|
||||
"lora_batch_layout",
|
||||
LoRABatchLayout.DP_LOCAL
|
||||
if TokenAxis.ATTN_DP in input_rows.sharded
|
||||
else LoRABatchLayout.TP_GLOBAL,
|
||||
)
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
tp_reduce_scatter,
|
||||
tp_slice,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
LayerResidual,
|
||||
StageRead,
|
||||
@@ -102,6 +103,7 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
# How the producer's output is written into the residual. An arrival
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
update: Optional[StageUpdate] = ADD
|
||||
transform: Optional[OutputTransform] = None
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
@@ -481,6 +483,7 @@ class FusedMlpInput(msgspec.Struct, frozen=True):
|
||||
# The group whose sum it completes.
|
||||
completes: SumGroup
|
||||
run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]]
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
|
||||
@@ -536,12 +539,12 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
|
||||
# Completes what the input owes, writes the previous stage's output into
|
||||
# the residual and reads this stage's input:
|
||||
# (hidden_states, residual, forward_batch, norm, context, **call).
|
||||
# (hidden_states, residual, forward_batch, norm, **call).
|
||||
prepare: Callable
|
||||
# The rows prepare hands the stage.
|
||||
input_rows: Layout
|
||||
# Moves the input onto the stage's rows after prepare, when prepare does
|
||||
# not: (hidden_states, forward_batch, context) -> hidden_states.
|
||||
# not: (hidden_states, forward_batch) -> hidden_states.
|
||||
input_move: Optional[Callable] = None
|
||||
# Hands the stage its input once it is on its rows:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
@@ -551,6 +554,7 @@ class StageEntry(msgspec.Struct, frozen=True):
|
||||
# Return the updated residual to the producer's rows for aux capture.
|
||||
capture_move: Optional[Callable] = None
|
||||
capture_move_allocates: bool = False
|
||||
preserves_residual: Optional[Callable] = None
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
input_sum: Optional[SumGroup] = None
|
||||
|
||||
@@ -619,6 +623,7 @@ class Boundary(msgspec.Struct, frozen=True):
|
||||
output_move: Optional[Callable] = None
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
@@ -682,6 +687,7 @@ def make_boundary(
|
||||
raise NotImplementedError("update capabilities require different input moves")
|
||||
return msgspec.structs.replace(
|
||||
first,
|
||||
preserves_residual=None,
|
||||
prepare=partial(
|
||||
_dispatch_consumer,
|
||||
paths={capability: path.prepare for capability, path in paths.items()},
|
||||
@@ -756,8 +762,24 @@ def _bind_consumer(
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=False,
|
||||
)
|
||||
preserves_residual = None
|
||||
if (
|
||||
edge.produced.always_leaves
|
||||
and edge.produced.group is SumGroup.ATTN_TP
|
||||
and isinstance(step, partial)
|
||||
and step.func is _mlp_input_without_dp
|
||||
and not step.keywords["gathers_residual"]
|
||||
):
|
||||
selected = step.keywords["fusions"]
|
||||
if selected:
|
||||
# Only the first reachable candidate can certify ownership; an
|
||||
# earlier custom candidate could otherwise mutate the residual.
|
||||
preserves_residual = next(
|
||||
f.preserves_residual for f in fusions if f.run == selected[0]
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
preserves_residual=preserves_residual,
|
||||
prepare=partial(
|
||||
_consumer_step,
|
||||
step=step,
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Bind one stage to predeclared boundaries and reusable batch paths."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property
|
||||
from typing import Callable, Optional, Protocol, Tuple
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
EdgeDecl,
|
||||
FusedMlpInput,
|
||||
StageDecl,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
make_boundary,
|
||||
make_output_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import OutputBoundary
|
||||
from sglang.srt.layers.communicator.fusions.allreduce import (
|
||||
attention_fusions,
|
||||
ffn_fusions,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
TokenAxis,
|
||||
_batch_shards_over_cp,
|
||||
_gathers_over_attention_cp,
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
|
||||
|
||||
class BoundaryFusions(Protocol):
|
||||
"""The fused kernels a fusion backend gives a layer, bound to it and tried
|
||||
before the layer's own: at its attention input (each takes what the previous
|
||||
layer left, the residual, the batch and a post-residual addition), at its
|
||||
FFN input, and at its FFN exit (each says what the exit does when its kernel
|
||||
takes the batch)."""
|
||||
|
||||
def attention_input(self, layer: "StagePlan") -> Tuple[Callable, ...]: ...
|
||||
|
||||
def ffn_input(self, layer: "StagePlan") -> Tuple["FusedMlpInput", ...]: ...
|
||||
|
||||
requires_local_reduction: bool
|
||||
|
||||
def can_defer_finalize(
|
||||
self, layer: "StagePlan", forward_batch: ForwardBatch
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
"""A MoE layer under attention CP whose tokens the steps cannot bring
|
||||
to it: under a prefill CP, one dispatched per DP shard under attention DP
|
||||
and GQA CP, and one on the TP group whose data-parallel groups are the CP
|
||||
ranks under DSA or MLA CP; and under attention DP, one on the TP group
|
||||
whose data-parallel groups are the CP ranks."""
|
||||
parallel = get_parallel()
|
||||
gqa = not _gathers_over_attention_cp()
|
||||
if moe_on_local_rows:
|
||||
if cp_shards and gqa and parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
|
||||
)
|
||||
elif parallel.moe_dp_size == parallel.attn_cp_size:
|
||||
if cp_shards and not gqa:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"DSA or MLA prefill CP"
|
||||
)
|
||||
if parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"attention DP and attention CP"
|
||||
)
|
||||
|
||||
|
||||
def _input_can_be_scattered() -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on TP without attention DP, a prefill CP, an a2a backend or
|
||||
a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
|
||||
known once the model is built."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and parallel.attn_dp_size == 1
|
||||
and not _generic_prefill_cp_shards_tokens()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not enable_moe_dense_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
class BatchVariant(Enum):
|
||||
ORDINARY = auto()
|
||||
CONTEXT_PARALLEL = auto()
|
||||
INPUT_SCATTERED = auto()
|
||||
SEQUENCE_PARALLEL = auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageEdges:
|
||||
"""One stage's incoming and outgoing declarations for one batch variant.
|
||||
|
||||
No adjacent stage object is required. Edges can come from a local model
|
||||
assembler, pipeline contract or branch adapter.
|
||||
"""
|
||||
|
||||
incoming: EdgeDecl
|
||||
outgoing: EdgeDecl
|
||||
handoff: Optional[Callable] = None
|
||||
cp_moves: object = None
|
||||
|
||||
|
||||
def _requires_branch_input(*args, **kwargs):
|
||||
raise RuntimeError("a prepared branch must enter through branch_input, not prepare")
|
||||
|
||||
|
||||
class StagePlan:
|
||||
"""Precomputed paths for one stage; never owns the neighbouring norm."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kind,
|
||||
norm,
|
||||
variants,
|
||||
*,
|
||||
enters_stack=False,
|
||||
prepared_input=False,
|
||||
terminal=False,
|
||||
fixed_output=False,
|
||||
is_sparse=False,
|
||||
qkv_latent_func=None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
enable_fused_ar_quant=False,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
residual_in_hidden=False,
|
||||
fusions=None,
|
||||
):
|
||||
self.kind = kind
|
||||
self.norm = norm
|
||||
self.variants = dict(variants)
|
||||
self.is_first_layer = enters_stack
|
||||
self.is_last_layer = terminal
|
||||
self.fixed_output = fixed_output
|
||||
self.is_sparse = is_sparse
|
||||
self.qkv_latent_func = qkv_latent_func
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.residual_in_hidden = residual_in_hidden
|
||||
self.fusions = fusions
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
||||
get_lora().enable_lora
|
||||
)
|
||||
self._fusion_rows = None
|
||||
carried = attention_fusions(self) if kind is StageKind.ATTENTION else ()
|
||||
self._carried_fusions = carried
|
||||
fused = ffn_fusions(self) if kind is StageKind.FFN else ()
|
||||
self._paths = {}
|
||||
for variant, edges in self.variants.items():
|
||||
if prepared_input:
|
||||
# The branch adapter moves an already-read input and forks its
|
||||
# stream. Binding a normal prepare would add or norm it twice.
|
||||
entry = StageEntry(
|
||||
prepare=_requires_branch_input,
|
||||
input_rows=edges.incoming.need.layout,
|
||||
)
|
||||
else:
|
||||
into = make_boundary(
|
||||
edges.incoming,
|
||||
fusions=fused,
|
||||
carried_fusions=carried,
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
cp_moves=edges.cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
entry = StageEntry(
|
||||
prepare=into.prepare,
|
||||
input_rows=into.input_rows,
|
||||
input_move=into.input_move,
|
||||
handoff=edges.handoff,
|
||||
fused=into.fused,
|
||||
capture_move=into.capture_move,
|
||||
capture_move_allocates=into.capture_move_allocates,
|
||||
preserves_residual=into.preserves_residual,
|
||||
input_sum=edges.incoming.produced.group
|
||||
if edges.incoming.produced.always_leaves
|
||||
else None,
|
||||
)
|
||||
out = (
|
||||
None
|
||||
if fixed_output
|
||||
else make_output_boundary(edges.outgoing, cp_moves=edges.cp_moves)
|
||||
)
|
||||
self._paths[variant] = BoundarySteps(
|
||||
attention=entry if kind is StageKind.ATTENTION else None,
|
||||
ffn=entry if kind is StageKind.FFN else None,
|
||||
ffn_output=edges.outgoing.produced,
|
||||
ffn_output_move=None if out is None else out.output_move,
|
||||
ffn_output_move_completes_sum=False
|
||||
if out is None
|
||||
else out.output_move_completes_sum,
|
||||
ffn_sum_is_movable=edges.outgoing.produced.group is not None,
|
||||
)
|
||||
self._steps = self._paths[BatchVariant.ORDINARY]
|
||||
self._cp_steps = self._paths.get(BatchVariant.CONTEXT_PARALLEL)
|
||||
self._input_scattered_steps = self._paths.get(BatchVariant.INPUT_SCATTERED)
|
||||
self._sp_steps = self._paths.get(BatchVariant.SEQUENCE_PARALLEL)
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
return self.variants[BatchVariant.ORDINARY].incoming.residual
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded
|
||||
|
||||
def _variant(self, forward_batch):
|
||||
if self._sp_steps is not None and get_forward().sp_active:
|
||||
return BatchVariant.SEQUENCE_PARALLEL
|
||||
if (
|
||||
self._input_scattered_steps is not None
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
return BatchVariant.INPUT_SCATTERED
|
||||
if self._cp_steps is not None and _batch_shards_over_cp(forward_batch):
|
||||
return BatchVariant.CONTEXT_PARALLEL
|
||||
return BatchVariant.ORDINARY
|
||||
|
||||
def _batch_steps(self, forward_batch):
|
||||
return self._paths[self._variant(forward_batch)]
|
||||
|
||||
def fusion_rows(self, forward_batch):
|
||||
if self._fusion_rows is not None:
|
||||
return self._fusion_rows[self._variant(forward_batch)]
|
||||
entry = self._batch_steps(forward_batch)
|
||||
return (entry.attention or entry.ffn).input_rows
|
||||
|
||||
def produced(self, kind, forward_batch):
|
||||
return self._batch_steps(forward_batch).ffn_output
|
||||
|
||||
@cached_property
|
||||
def output(self):
|
||||
return OutputBoundary(self)
|
||||
|
||||
def _branch_rows(self, forward_batch):
|
||||
if self._variant(forward_batch) is not BatchVariant.ORDINARY:
|
||||
raise NotImplementedError("branch transport requires ordinary batch rows")
|
||||
edges = self.variants[BatchVariant.ORDINARY]
|
||||
return (
|
||||
edges.incoming.need.layout,
|
||||
edges.incoming.residual_to,
|
||||
edges.outgoing.need.layout,
|
||||
)
|
||||
|
||||
|
||||
def make_stage(
|
||||
declaration: StageDecl,
|
||||
*,
|
||||
kind: StageKind,
|
||||
norm,
|
||||
incoming: EdgeDecl,
|
||||
outgoing: EdgeDecl,
|
||||
variants=None,
|
||||
handoff=None,
|
||||
**options,
|
||||
) -> StageCommunicator:
|
||||
"""Build one real stage from its own declaration and adjacent edges.
|
||||
|
||||
The optional variants supply the same stage's CP/SP/input-scattered
|
||||
declarations. Neither a paired stage nor its layer facts/norm is needed.
|
||||
"""
|
||||
if incoming.need != declaration.input or outgoing.produced != declaration.output:
|
||||
raise ValueError("stage declaration disagrees with its boundaries")
|
||||
paths = dict(variants or {})
|
||||
paths[BatchVariant.ORDINARY] = StageEdges(incoming, outgoing, handoff)
|
||||
return StageCommunicator(StagePlan(kind, norm, paths, **options), kind, norm)
|
||||
@@ -0,0 +1,521 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Producer output choices, reduction scopes, and completion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
SumGroup,
|
||||
_ffn_has_tokens,
|
||||
_sum_group,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
_all_reduce_then_to_local_tokens,
|
||||
_redistribute_output,
|
||||
_reduce_and_redistribute_output_max_len,
|
||||
_reduce_and_redistribute_output_varlen,
|
||||
_to_local_tokens,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import (
|
||||
UnreducedOutput,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
can_use_dp_reduce_scatter,
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
post_experts_reduction_group,
|
||||
post_experts_sum_is_one_all_reduce,
|
||||
should_use_dp_reduce_scatterv,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
)
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_step(
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings an FFN output gathered over attention
|
||||
DP back to this rank's tokens when the FFN leaves its sum to it (see
|
||||
StageOutput); None when the FFN reduces the output and only a scatter
|
||||
remains."""
|
||||
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
|
||||
return _reduce_and_redistribute_output_varlen
|
||||
if (
|
||||
leaves_for_reduce_scatter
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and can_use_dp_reduce_scatter()
|
||||
):
|
||||
return _reduce_and_redistribute_output_max_len
|
||||
return None
|
||||
|
||||
|
||||
class OutputBoundary:
|
||||
"""Choose and finish one producer's output; the batch owns its residual."""
|
||||
|
||||
def __init__(self, plan):
|
||||
self.plan = plan
|
||||
|
||||
def postprocess_layer(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
"""Move a complete output from the operation-scheduled producer path."""
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
hidden_states, residual = self._complete_ffn_output_now(
|
||||
hidden_states,
|
||||
stream.residual if stream is not None else residual,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=None,
|
||||
)
|
||||
return self._leave_ffn_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
stream,
|
||||
self.plan._batch_steps(forward_batch).ffn_output.update,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _leave_ffn_output(hidden_states, residual, stream, update, declared_sum=None):
|
||||
if stream is None:
|
||||
return hidden_states, residual
|
||||
stream.residual = residual
|
||||
if update.at_producer:
|
||||
return stream.write(hidden_states), stream
|
||||
return stream.leave(hidden_states, update, declared_sum=declared_sum), stream
|
||||
|
||||
@staticmethod
|
||||
def _declared_ffn_sum(steps, skipped_reduction):
|
||||
# The input-scattered path leaves the sum for the next input's TP
|
||||
# reduce-scatter. Other skip paths complete it in the output move.
|
||||
if (
|
||||
skipped_reduction
|
||||
and not steps.returns_over_dp
|
||||
and not steps.ffn_output_move_completes_sum
|
||||
):
|
||||
return SumGroup.TP
|
||||
return None
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
the next layer's input also writes the output into the residual."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
return steps.returns_over_dp and not steps.ffn_output.update.at_producer
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings this layer's FFN output back to this
|
||||
rank's tokens under attention DP; None when the base postprocess would
|
||||
only scatter, or does not move tokens."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if not steps.returns_over_dp:
|
||||
return None
|
||||
return _reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
|
||||
)
|
||||
|
||||
def _ffn_leaves_sum_to_reduce_scatter(
|
||||
self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if dp_step is not None:
|
||||
return True
|
||||
if not steps.ffn_output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if steps.ffn_output_move_completes_sum:
|
||||
return True
|
||||
return get_attn_tp_context().input_scattered and not self.plan.is_last_layer
|
||||
|
||||
def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator:
|
||||
"""The group this layer's FFN output owes its sum over: the MoE output's
|
||||
group on a sparse layer, the TP group a dense MLP reduces over."""
|
||||
return _sum_group(self.plan._batch_steps(forward_batch).ffn_output.group)
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
|
||||
"""Decide once, before the FFN runs, what it skips and what completes its
|
||||
output: the next layer's input, or this layer's postprocess step."""
|
||||
dp_step = self._postprocess_dp_step(forward_batch)
|
||||
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
|
||||
forward_batch, dp_step
|
||||
)
|
||||
complete_now = partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if not steps.ffn_output.leaves_for_next_layer:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
)
|
||||
# Producers declare remaining work independently of the kernel chosen
|
||||
# by the consumer. Every handoff also carries an unfused completion.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
group = self.ffn_reduction_group(forward_batch)
|
||||
if steps.returns_over_dp:
|
||||
# Under attention DP the next layer also brings the sum back to
|
||||
# this rank's tokens.
|
||||
wrap = partial(
|
||||
UnreducedOutput,
|
||||
reduce_and_redistribute=partial(
|
||||
_all_reduce_then_to_local_tokens, group, forward_batch
|
||||
),
|
||||
)
|
||||
else:
|
||||
wrap = partial(UnreducedOutput, group=group)
|
||||
complete = partial(_leave_to_next_layer, wrap)
|
||||
elif (
|
||||
dp_step is not None
|
||||
and not self.plan.is_last_layer
|
||||
and self._local_token_move_can_go_to_next_layer(forward_batch)
|
||||
):
|
||||
complete = partial(
|
||||
_leave_to_next_layer,
|
||||
partial(
|
||||
UnreducedOutput,
|
||||
reduce_and_redistribute=partial(
|
||||
_to_local_tokens, dp_step, forward_batch
|
||||
),
|
||||
),
|
||||
)
|
||||
else:
|
||||
complete = complete_now
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=defer_moe_finalize,
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete,
|
||||
)
|
||||
|
||||
def _complete_ffn_output_now(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
*,
|
||||
forward_batch: ForwardBatch,
|
||||
dp_step: Optional[Callable],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the move back to where the next layer reads the FFN output, then
|
||||
the write-back into the residual for a layer that does it itself."""
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
if steps.ffn_output.transform is not None:
|
||||
hidden_states = steps.ffn_output.transform.apply(hidden_states)
|
||||
if steps.returns_over_dp:
|
||||
hidden_states = _to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
update = steps.ffn_output.update
|
||||
if residual is not None and update.at_producer:
|
||||
hidden_states = update.update(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def mixer_exit(self, forward_batch: ForwardBatch) -> MixerExit:
|
||||
"""Decide once whether this stage's mixer (an attention-like stage)
|
||||
skips its output all-reduce. Use the result as a context manager around
|
||||
the mixer, then call ``finish``."""
|
||||
return MixerExit(self, forward_batch)
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> FfnExit:
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch)
|
||||
|
||||
def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if (
|
||||
(
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.requires_local_reduction
|
||||
)
|
||||
or is_enable_moe_cp_allgather()
|
||||
or not self.plan._batch_steps(forward_batch).ffn_sum_is_movable
|
||||
):
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
# two disjoint groups; post_experts_all_reduce() merges them into one
|
||||
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
|
||||
# When merging is blocked, no single group covers both, so fusion stays off.
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.moe_ep_size > 1
|
||||
and parallel.moe_tp_size > 1
|
||||
and not can_merge_post_experts_all_reduce()
|
||||
):
|
||||
return False
|
||||
|
||||
if (
|
||||
is_dp_attention_enabled()
|
||||
and self.plan._speculative_algo is not None
|
||||
and self.plan._speculative_algo.is_eagle()
|
||||
):
|
||||
return False
|
||||
|
||||
return not get_attn_tp_context().input_scattered
|
||||
|
||||
def _ffn_sum_moves_to_next_layer(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
mlp_reduce_scatter: bool,
|
||||
dp_step: Optional[Callable],
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
input when no fused kernel takes it: when the next layer would run the
|
||||
same all-reduce the FFN itself would have."""
|
||||
return (
|
||||
get_parallel().tp_size > 1
|
||||
and not self.plan.is_last_layer
|
||||
and self._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
and _can_defer_ffn_reduction(forward_batch, self.plan)
|
||||
and not mlp_reduce_scatter
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
and (
|
||||
not is_dp_attention_enabled()
|
||||
or (
|
||||
self._local_token_move_can_go_to_next_layer(forward_batch)
|
||||
and dp_step is None
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool:
|
||||
"""Admit ordinary single-sum outputs and the existing fused LoRA path.
|
||||
|
||||
A fused-kernel fallback completes the partial-output contract selected
|
||||
before the producer ran, without recomputing the producer's LoRA path.
|
||||
"""
|
||||
if not _ffn_has_tokens(forward_batch):
|
||||
return False
|
||||
if post_experts_sum_is_one_all_reduce():
|
||||
return True
|
||||
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
|
||||
# when the sum is deferred. Preserve these fused paths, but keep their
|
||||
# producer-side reduction order when no fused consumer is enabled.
|
||||
if not (
|
||||
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
and post_experts_reduction_group() is get_parallel().tp_group
|
||||
):
|
||||
return False
|
||||
if (
|
||||
boundary is not None
|
||||
and boundary.fusions is not None
|
||||
and boundary.fusions.can_defer_all_reduce(boundary, forward_batch)
|
||||
):
|
||||
return True
|
||||
if apply_flashinfer_allreduce_fusion(forward_batch.input_ids.shape[0]):
|
||||
return True
|
||||
# Aiter also checks width and bytes, so use the actual residual storage.
|
||||
residual = forward_batch.residual_stream.residual
|
||||
return residual is not None and apply_aiter_all_reduce_fusion(
|
||||
residual, forward_batch
|
||||
)
|
||||
|
||||
|
||||
class FfnCompletion(msgspec.Struct, frozen=True):
|
||||
"""One FFN's reduction decision. The flags are published while the FFN runs;
|
||||
``complete(hidden_states, residual)`` then either wraps the output for the
|
||||
next layer's input to complete, or runs this layer's postprocess step."""
|
||||
|
||||
defer_moe_finalize: bool
|
||||
fuse_mlp_allreduce: bool
|
||||
mlp_reduce_scatter: bool
|
||||
complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
|
||||
|
||||
|
||||
def _leave_to_next_layer(
|
||||
wrap: Callable[[torch.Tensor], UnreducedOutput],
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> Tuple[UnreducedOutput, torch.Tensor]:
|
||||
return wrap(hidden_states), residual
|
||||
|
||||
|
||||
class MixerExit:
|
||||
"""The scope that publishes a mixer's decision while it runs: inside the
|
||||
``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its
|
||||
row-parallel output projection to skip the all-reduce. It skips when the
|
||||
stage's output always leaves its sum (to an FFN stage, which completes it in
|
||||
its input), and when its output declaration permits deferring the sum. The
|
||||
consumer decides whether to fuse its completion with the input norm."""
|
||||
|
||||
__slots__ = (
|
||||
"skips_reduction",
|
||||
"_hands_on",
|
||||
"_scope",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
|
||||
produced = communicator.plan._batch_steps(forward_batch).ffn_output
|
||||
self._stream = None
|
||||
self._update = produced.update
|
||||
self._declared_sum = produced.group if produced.always_leaves else None
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
)
|
||||
self.skips_reduction = produced.always_leaves or self._hands_on
|
||||
self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction)
|
||||
|
||||
def __enter__(self) -> MixerExit:
|
||||
self._scope.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(self, hidden_states: torch.Tensor, residual=None):
|
||||
"""The mixer's output: its partial sum to the FFN stage after it, as a
|
||||
value that owes the sum to a mixer after it, or complete."""
|
||||
if self._stream is not None:
|
||||
residual = self._stream
|
||||
if self._hands_on:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
if isinstance(residual, ResidualStream):
|
||||
output = residual.leave(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
)
|
||||
return output if self._stream is not None else (output, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FfnExit:
|
||||
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
|
||||
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
|
||||
``defer_moe_finalize`` on ``get_forward()``."""
|
||||
|
||||
__slots__ = (
|
||||
"communicator",
|
||||
"forward_batch",
|
||||
"defer_moe_finalize",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"_complete",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_scope",
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: OutputBoundary, forward_batch: ForwardBatch):
|
||||
self._stream = None
|
||||
self.communicator = communicator
|
||||
self.forward_batch = forward_batch
|
||||
completion = communicator._select_ffn_completion(forward_batch)
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
||||
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
||||
self._complete = completion.complete
|
||||
steps = communicator.plan._batch_steps(forward_batch)
|
||||
self._update = steps.ffn_output.update
|
||||
self._declared_sum = communicator._declared_ffn_sum(
|
||||
steps, completion.mlp_reduce_scatter
|
||||
)
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
defer_moe_finalize=self.defer_moe_finalize,
|
||||
)
|
||||
|
||||
def __enter__(self) -> FfnExit:
|
||||
self._scope.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
|
||||
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input, or postprocess."""
|
||||
if self._stream is not None:
|
||||
residual = self._stream
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
residual = stream.residual
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
else:
|
||||
hidden_states, residual = self._complete(hidden_states, residual)
|
||||
result = self.communicator._leave_ffn_output(
|
||||
hidden_states, residual, stream, self._update, self._declared_sum
|
||||
)
|
||||
return result[0] if self._stream is not None else result
|
||||
@@ -0,0 +1,705 @@
|
||||
"""Construct one computing stage and its transport from a predecessor contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Mapping, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
EdgeDecl,
|
||||
StageDecl,
|
||||
StageInput,
|
||||
StageKind,
|
||||
StageOutput,
|
||||
_cp_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.construction import (
|
||||
BatchVariant,
|
||||
StageEdges,
|
||||
StagePlan,
|
||||
_input_can_be_scattered,
|
||||
_refuse_uncovered_cp_moe,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_gathers_over_attention_cp,
|
||||
_generic_prefill_cp_shards_tokens,
|
||||
enable_moe_dense_fully_dp,
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
_hand_qkv_hook_its_input,
|
||||
_hand_scattered_input_to_attention,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
)
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
|
||||
def _variants(*, ordinary_only=False):
|
||||
yield BatchVariant.ORDINARY
|
||||
if ordinary_only:
|
||||
return
|
||||
if _generic_prefill_cp_shards_tokens():
|
||||
yield BatchVariant.CONTEXT_PARALLEL
|
||||
if _input_can_be_scattered():
|
||||
yield BatchVariant.INPUT_SCATTERED
|
||||
if layernorm_sp.layernorm_sp_enabled():
|
||||
yield BatchVariant.SEQUENCE_PARALLEL
|
||||
|
||||
|
||||
def _rows(variant):
|
||||
axes = token_axis_sizes(cp_active=variant is BatchVariant.CONTEXT_PARALLEL)
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, TokenAxis.ATTN_TP_SCATTER, axis_sizes=axes
|
||||
)
|
||||
return axes, attention, local, Layout.sharded_over(axis_sizes=axes)
|
||||
|
||||
|
||||
def _ffn_decl(
|
||||
variant,
|
||||
*,
|
||||
sparse,
|
||||
terminal=False,
|
||||
next_sparse=False,
|
||||
output=None,
|
||||
read=NORM_READ,
|
||||
update=ADD,
|
||||
local_dense=True,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
reduction = get_exec().comm.boundary_reduction
|
||||
if reduction not in ("ar", "rs", "rsv", "rs+rsv"):
|
||||
raise ValueError(
|
||||
"boundary_reduction must be resolved before model construction"
|
||||
)
|
||||
can_move_output = output is None or output.before_reduce_scatter
|
||||
use_reduce_scatter = reduction in ("rs", "rs+rsv") and can_move_output
|
||||
use_reduce_scatterv = reduction in ("rsv", "rs+rsv") and can_move_output
|
||||
cp_shards = _generic_prefill_cp_shards_tokens()
|
||||
axes, attention, local, full = _rows(variant)
|
||||
on_local = (
|
||||
is_moe_input_scattered_across_dp_ranks()
|
||||
if sparse
|
||||
else (local_dense and enable_moe_dense_fully_dp())
|
||||
)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_refuse_uncovered_cp_moe(on_local, cp_shards)
|
||||
on_cp = (
|
||||
sparse
|
||||
and parallel.attn_cp_size > 1
|
||||
and parallel.moe_dp_size == parallel.attn_cp_size
|
||||
and not _gathers_over_attention_cp()
|
||||
)
|
||||
group = SumGroup.MOE_OUTPUT if sparse else SumGroup.TP
|
||||
if variant is BatchVariant.SEQUENCE_PARALLEL:
|
||||
return (
|
||||
StageDecl(
|
||||
StageInput(
|
||||
full,
|
||||
gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}),
|
||||
read=read,
|
||||
),
|
||||
StageOutput(local, update=update, transform=output),
|
||||
),
|
||||
local,
|
||||
local,
|
||||
)
|
||||
if variant is BatchVariant.INPUT_SCATTERED:
|
||||
scattered_residual = update.at_producer
|
||||
residual = local if scattered_residual else attention
|
||||
returned = local if scattered_residual and not terminal else attention
|
||||
return (
|
||||
StageDecl(
|
||||
StageInput(full, read=read),
|
||||
StageOutput(
|
||||
attention,
|
||||
group=group,
|
||||
leaves_for_reduce_scatter=use_reduce_scatter
|
||||
and (scattered_residual or not terminal),
|
||||
update=update,
|
||||
transform=output,
|
||||
),
|
||||
),
|
||||
residual,
|
||||
returned,
|
||||
)
|
||||
if cp_shards and _cp_moves().reduce_scatter is not None:
|
||||
may_leave = variant is not BatchVariant.CONTEXT_PARALLEL
|
||||
may_scatter = True
|
||||
elif cp_shards or parallel.attn_dp_size > 1:
|
||||
may_leave = may_scatter = parallel.attn_cp_size == 1 or on_cp
|
||||
else:
|
||||
may_leave = may_scatter = True
|
||||
rows = (
|
||||
local
|
||||
if on_local
|
||||
else Layout.sharded_over(
|
||||
*((TokenAxis.ATTN_CP,) if on_cp else ()), axis_sizes=axes
|
||||
)
|
||||
)
|
||||
output = (
|
||||
StageOutput(rows, update=update, transform=output)
|
||||
if on_local
|
||||
else StageOutput(
|
||||
rows,
|
||||
group=group,
|
||||
leaves_for_next_layer=may_leave
|
||||
and not terminal
|
||||
and not update.at_producer
|
||||
and update.can_defer_across_layers
|
||||
and output is None,
|
||||
leaves_for_reduce_scatter=use_reduce_scatter and may_scatter,
|
||||
leaves_for_reduce_scatterv=use_reduce_scatterv and may_leave,
|
||||
update=update,
|
||||
transform=output,
|
||||
)
|
||||
)
|
||||
gathers_for_tbo = (
|
||||
enable_moe_dense_fully_dp()
|
||||
and get_exec().overlap.enable_two_batch_overlap
|
||||
and not sparse
|
||||
and bool(next_sparse)
|
||||
)
|
||||
returned = local if on_local and not terminal and not gathers_for_tbo else attention
|
||||
return (
|
||||
StageDecl(StageInput(rows, read=read), output),
|
||||
local if on_local else attention,
|
||||
returned,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageDeclaration:
|
||||
"""A computation and its input source, without an execution plan.
|
||||
|
||||
``previous`` consumes a producer's output. ``prepared_from`` reuses an
|
||||
already-read input for a branch; the two sources are mutually exclusive.
|
||||
Sources may be reconstructed locally, including across pipeline ranks.
|
||||
"""
|
||||
|
||||
kind: StageKind
|
||||
read: StageRead
|
||||
update: StageUpdate
|
||||
sparse: bool = False
|
||||
terminal: bool = False
|
||||
next_sparse: bool = False
|
||||
output: Optional[OutputTransform] = None
|
||||
ordinary_only: bool = False
|
||||
# A mixer with its own exit publishes reduction decisions before compute.
|
||||
mixer_exit: bool = False
|
||||
next_kind: Optional[StageKind] = None
|
||||
return_to_attention: bool = False
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.previous is not None and self.prepared_from is not None:
|
||||
raise ValueError("choose a previous output or a prepared branch input")
|
||||
for source in (self.previous, self.prepared_from):
|
||||
if source is not None and not isinstance(source, StageDeclaration):
|
||||
raise TypeError("stage sources must be declarations")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageConnection:
|
||||
"""The producer's exit and consumer's entry for each batch variant.
|
||||
|
||||
Immutable, norm-free declarations. Independently constructed equivalent
|
||||
connections work across pipeline partitions; object identity is irrelevant.
|
||||
"""
|
||||
|
||||
producer: Optional[StageDeclaration]
|
||||
consumer: Optional[StageDeclaration]
|
||||
exits: Mapping[BatchVariant, EdgeDecl]
|
||||
entries: Mapping[BatchVariant, EdgeDecl]
|
||||
|
||||
def __post_init__(self):
|
||||
object.__setattr__(self, "exits", MappingProxyType(dict(self.exits)))
|
||||
object.__setattr__(self, "entries", MappingProxyType(dict(self.entries)))
|
||||
|
||||
|
||||
def declare_attn(
|
||||
*,
|
||||
previous: Optional[StageDeclaration] = None,
|
||||
prepared_from: Optional[StageDeclaration] = None,
|
||||
read=NORM_QUANT_READ,
|
||||
update=ADD,
|
||||
terminal=False,
|
||||
ordinary_only=False,
|
||||
mixer_exit=False,
|
||||
next_kind=None,
|
||||
):
|
||||
return StageDeclaration(
|
||||
StageKind.ATTENTION,
|
||||
read,
|
||||
update,
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
ordinary_only=ordinary_only,
|
||||
mixer_exit=mixer_exit,
|
||||
next_kind=next_kind,
|
||||
)
|
||||
|
||||
|
||||
def declare_ffn(
|
||||
*,
|
||||
previous: Optional[StageDeclaration] = None,
|
||||
prepared_from: Optional[StageDeclaration] = None,
|
||||
sparse=False,
|
||||
read=NORM_READ,
|
||||
update=ADD,
|
||||
terminal=False,
|
||||
next_sparse=False,
|
||||
output=None,
|
||||
ordinary_only=False,
|
||||
return_to_attention=False,
|
||||
):
|
||||
return StageDeclaration(
|
||||
StageKind.FFN,
|
||||
read,
|
||||
update,
|
||||
sparse=sparse,
|
||||
previous=previous,
|
||||
prepared_from=prepared_from,
|
||||
terminal=terminal,
|
||||
next_sparse=next_sparse,
|
||||
output=output,
|
||||
ordinary_only=ordinary_only,
|
||||
return_to_attention=return_to_attention,
|
||||
)
|
||||
|
||||
|
||||
def _resolve(stage, variant):
|
||||
axes, attention, local, full = _rows(variant)
|
||||
if stage.kind is StageKind.FFN:
|
||||
if stage.update.at_producer:
|
||||
if stage.sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
)
|
||||
declaration, residual, returned = _ffn_decl(
|
||||
variant,
|
||||
sparse=stage.sparse,
|
||||
terminal=stage.terminal,
|
||||
next_sparse=stage.next_sparse,
|
||||
output=stage.output,
|
||||
read=stage.read,
|
||||
update=stage.update,
|
||||
local_dense=not stage.return_to_attention,
|
||||
)
|
||||
if stage.return_to_attention:
|
||||
returned = attention
|
||||
return declaration, residual, returned
|
||||
sp = variant is BatchVariant.SEQUENCE_PARALLEL
|
||||
scattered = variant is BatchVariant.INPUT_SCATTERED
|
||||
gathers = (
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if not stage.mixer_exit and (sp or scattered or _use_ag_after_qlora)
|
||||
else frozenset()
|
||||
)
|
||||
owes = not sp and axes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
declaration = StageDecl(
|
||||
StageInput(attention, gathers_itself=gathers, read=stage.read),
|
||||
StageOutput(
|
||||
local if sp else attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
always_leaves=owes
|
||||
and (not stage.mixer_exit or stage.next_kind is StageKind.FFN),
|
||||
leaves_for_next_layer=owes
|
||||
and stage.mixer_exit
|
||||
and stage.next_kind is StageKind.ATTENTION,
|
||||
update=stage.update,
|
||||
),
|
||||
)
|
||||
return declaration, None, attention
|
||||
|
||||
|
||||
def _connect(producer, consumer, *, residual_from=None):
|
||||
"""Resolve producer and consumer declarations; None denotes an external endpoint.
|
||||
|
||||
A missing producer is the stack input; a missing consumer is a handoff
|
||||
whose next read is bound separately (or the terminal output).
|
||||
``residual_from`` names the boundary that placed the producer's residual.
|
||||
It is needed for attention, whose residual can stay on finer rows than its
|
||||
compute input. No executable stage participates in this construction.
|
||||
"""
|
||||
if producer is None and consumer is None:
|
||||
raise ValueError("a boundary needs at least one declared side")
|
||||
before, after = producer, consumer
|
||||
ordinary_only = any(s.ordinary_only for s in (before, after) if s is not None)
|
||||
exits, entries = {}, {}
|
||||
for variant in _variants(ordinary_only=ordinary_only):
|
||||
_, attention, local, _ = _rows(variant)
|
||||
if before is None:
|
||||
rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention
|
||||
owes = variant is BatchVariant.INPUT_SCATTERED
|
||||
arrived = StageOutput(
|
||||
rows,
|
||||
group=SumGroup.TP if owes else None,
|
||||
always_leaves=owes,
|
||||
update=None,
|
||||
)
|
||||
residual = rows
|
||||
capabilities = (True,)
|
||||
else:
|
||||
decl, during, returned = _resolve(before, variant)
|
||||
if during is None:
|
||||
if residual_from is not None:
|
||||
if residual_from.consumer != producer:
|
||||
raise ValueError("residual source must enter the producer")
|
||||
during = residual_from.entries[variant].residual_to
|
||||
elif before.mixer_exit:
|
||||
during = attention
|
||||
else:
|
||||
raise ValueError(
|
||||
"attention output needs its incoming residual placement"
|
||||
)
|
||||
exits[variant] = EdgeDecl(
|
||||
decl.output, StageInput(returned), during, returned
|
||||
)
|
||||
if before.kind is StageKind.ATTENTION and not before.mixer_exit:
|
||||
arrived, residual, capabilities = decl.output, during, ()
|
||||
else:
|
||||
owes = (
|
||||
variant is BatchVariant.INPUT_SCATTERED
|
||||
and not before.update.at_producer
|
||||
)
|
||||
carries = (before.mixer_exit or before.return_to_attention) and (
|
||||
decl.output.always_leaves or decl.output.leaves_for_next_layer
|
||||
)
|
||||
arrived = StageOutput(
|
||||
returned,
|
||||
group=decl.output.group
|
||||
if carries
|
||||
else (SumGroup.TP if owes else None),
|
||||
always_leaves=decl.output.always_leaves if carries else owes,
|
||||
leaves_for_next_layer=decl.output.leaves_for_next_layer
|
||||
if carries
|
||||
else False,
|
||||
update=None,
|
||||
)
|
||||
residual, capabilities = returned, (before.update.adds_plainly,)
|
||||
if after is None:
|
||||
continue
|
||||
decl, during, _ = _resolve(after, variant)
|
||||
if after.kind is StageKind.ATTENTION:
|
||||
during = (
|
||||
Layout(residual.sharded | {TokenAxis.ATTN_TP_SCATTER})
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and arrived.always_leaves
|
||||
and arrived.update is None
|
||||
and not after.mixer_exit
|
||||
else residual
|
||||
)
|
||||
elif after.return_to_attention:
|
||||
during = (
|
||||
decl.input.layout
|
||||
if residual.sharded <= decl.input.layout.sharded
|
||||
else residual
|
||||
)
|
||||
joins = (
|
||||
variant is BatchVariant.INPUT_SCATTERED
|
||||
and arrived.update is not None
|
||||
and arrived.update.adds_plainly
|
||||
and after.kind is StageKind.FFN
|
||||
)
|
||||
edge = EdgeDecl(
|
||||
arrived,
|
||||
decl.input,
|
||||
residual,
|
||||
during,
|
||||
residual_joins_sum=joins,
|
||||
update_capabilities=capabilities,
|
||||
)
|
||||
entries[variant] = edge
|
||||
if (
|
||||
before is not None
|
||||
and before.kind is StageKind.ATTENTION
|
||||
and not before.mixer_exit
|
||||
):
|
||||
exits[variant] = edge
|
||||
return StageConnection(producer, consumer, exits, entries)
|
||||
|
||||
|
||||
def _fork_input(prepared, consumer):
|
||||
"""Place an already-read input onto a branch's computation rows.
|
||||
|
||||
The branch adapter performs the move and forks the stream. It does not
|
||||
execute the branch's input norm again.
|
||||
"""
|
||||
entries = {}
|
||||
for variant in _variants(ordinary_only=consumer.ordinary_only):
|
||||
source = prepared.entries[variant]
|
||||
declaration, during, _ = _resolve(consumer, variant)
|
||||
entries[variant] = EdgeDecl(
|
||||
StageOutput(source.need.layout, update=None),
|
||||
declaration.input,
|
||||
source.residual_to,
|
||||
during,
|
||||
update_capabilities=(True,),
|
||||
)
|
||||
return StageConnection(prepared.consumer, consumer, {}, entries)
|
||||
|
||||
|
||||
def _incoming(stage):
|
||||
if stage.prepared_from is not None:
|
||||
return _fork_input(_incoming(stage.prepared_from), stage)
|
||||
previous = stage.previous
|
||||
# A normal attention may retain a finer residual than its output rows.
|
||||
# FFN and mixer exits declare their returned residual placement directly.
|
||||
source = (
|
||||
_incoming(previous)
|
||||
if previous is not None
|
||||
and previous.kind is StageKind.ATTENTION
|
||||
and not previous.mixer_exit
|
||||
else None
|
||||
)
|
||||
return _connect(
|
||||
previous,
|
||||
stage,
|
||||
residual_from=source,
|
||||
)
|
||||
|
||||
|
||||
def _connections(stage, following=None):
|
||||
if following is not None and following.previous != stage:
|
||||
raise ValueError("the following declaration must consume this stage's output")
|
||||
incoming = _incoming(stage)
|
||||
outgoing = _connect(
|
||||
stage,
|
||||
following,
|
||||
residual_from=incoming,
|
||||
)
|
||||
return incoming, outgoing
|
||||
|
||||
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
if incoming.entries.keys() != outgoing.exits.keys():
|
||||
raise ValueError("incoming and outgoing batch variants disagree")
|
||||
if declaration.update.at_producer:
|
||||
if declaration.sparse and sparse_moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"MHC does not support a MoE gathered over the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHC with input-scattered attention under attention CP"
|
||||
)
|
||||
variants = {}
|
||||
for variant, edge in incoming.entries.items():
|
||||
if (
|
||||
declaration.update.at_producer
|
||||
and TokenAxis.ATTN_CP
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
handoff = None
|
||||
if declaration.kind is StageKind.ATTENTION:
|
||||
handoff = (
|
||||
_hand_scattered_input_to_attention
|
||||
if variant is BatchVariant.INPUT_SCATTERED
|
||||
and not declaration.update.adds_plainly
|
||||
else _hand_qkv_hook_its_input
|
||||
)
|
||||
moves = _cp_moves() if variant is BatchVariant.CONTEXT_PARALLEL else None
|
||||
variants[variant] = StageEdges(edge, outgoing.exits[variant], handoff, moves)
|
||||
plan = StagePlan(
|
||||
declaration.kind,
|
||||
norm,
|
||||
variants,
|
||||
enters_stack=incoming.producer is None,
|
||||
prepared_input=declaration.prepared_from is not None,
|
||||
terminal=declaration.terminal,
|
||||
fixed_output=declaration.kind is StageKind.ATTENTION
|
||||
and not declaration.mixer_exit,
|
||||
is_sparse=declaration.sparse,
|
||||
**options,
|
||||
)
|
||||
if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None:
|
||||
# Layout eligibility comes from the connected consumer, not a mutable
|
||||
# link to its execution plan. Kernel binding remains consumer-owned.
|
||||
from sglang.srt.layers.communicator.boundary import make_boundary
|
||||
|
||||
plan._fusion_rows = {
|
||||
v: make_boundary(
|
||||
edge,
|
||||
cp_moves=_cp_moves() if v is BatchVariant.CONTEXT_PARALLEL else None,
|
||||
force_layernorm_before_gather=options.get(
|
||||
"force_layernorm_before_dp_gather", False
|
||||
),
|
||||
).input_rows
|
||||
for v, edge in outgoing.entries.items()
|
||||
}
|
||||
return StageCommunicator(plan, declaration.kind, norm, declaration=declaration)
|
||||
|
||||
|
||||
def make_attn_stage(
|
||||
*,
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
qkv_latent_func=None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
enable_fused_ar_quant=False,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
residual_in_hidden=False,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this attention's boundaries, then bind its norm and hooks.
|
||||
|
||||
``following`` supplies the consumer declaration for a stage inside a
|
||||
layer. With no following stage, the output is handed across a layer or
|
||||
stack boundary; the receiver binds its own read independently.
|
||||
"""
|
||||
if declaration.kind is not StageKind.ATTENTION:
|
||||
raise TypeError("make_attn_stage requires an attention declaration")
|
||||
incoming, outgoing = _connections(declaration, following)
|
||||
return _bind_stage(
|
||||
declaration,
|
||||
norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
|
||||
residual_in_hidden=residual_in_hidden,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
|
||||
def make_ffn_stage(
|
||||
*,
|
||||
declaration,
|
||||
norm,
|
||||
following: Optional[StageDeclaration] = None,
|
||||
force_layernorm_before_dp_gather=False,
|
||||
fusions=None,
|
||||
):
|
||||
"""Resolve this FFN's boundaries and bind its norm; EP stays inside compute."""
|
||||
if declaration.kind is not StageKind.FFN:
|
||||
raise TypeError("make_ffn_stage requires an FFN declaration")
|
||||
incoming, outgoing = _connections(declaration, following)
|
||||
return _bind_stage(
|
||||
declaration,
|
||||
norm,
|
||||
incoming,
|
||||
outgoing,
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
residual_in_hidden=declaration.update.at_producer,
|
||||
fusions=fusions,
|
||||
)
|
||||
|
||||
|
||||
def make_stages(*stages, previous=None, prepared_from=None, terminal=False):
|
||||
"""Bind a local linear sequence and return its independent boundaries.
|
||||
|
||||
Each item is ``(declaration, norm)`` or ``(declaration, norm, options)``.
|
||||
Options are the stage-specific keyword arguments of ``make_attn_stage``
|
||||
or ``make_ffn_stage``; they are consumed only during construction.
|
||||
Declarations in the sequence have no input source: this function connects
|
||||
them in order. Only the final boundary is terminal when requested.
|
||||
|
||||
``previous`` describes an external producer's output. ``prepared_from``
|
||||
describes an already-read branch input, obtainable from the source
|
||||
boundary's immutable ``declaration``. Neither takes an executable boundary.
|
||||
No sequence object or runtime routing is retained.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("make_stages needs at least one stage")
|
||||
if previous is not None and prepared_from is not None:
|
||||
raise ValueError("choose a previous output or a prepared branch input")
|
||||
declarations = []
|
||||
bindings = []
|
||||
for index, item in enumerate(stages):
|
||||
if len(item) not in (2, 3):
|
||||
raise ValueError("a stage needs (declaration, norm[, options])")
|
||||
declaration, norm = item[:2]
|
||||
if not isinstance(declaration, StageDeclaration):
|
||||
raise TypeError("make_stages requires stage declarations")
|
||||
if declaration.previous is not None or declaration.prepared_from is not None:
|
||||
raise ValueError("pass input sources to make_stages, not its declarations")
|
||||
if declaration.terminal:
|
||||
raise ValueError("pass terminal to make_stages, not its declarations")
|
||||
declaration = replace(
|
||||
declaration,
|
||||
previous=previous if index == 0 else declarations[-1],
|
||||
prepared_from=prepared_from if index == 0 else None,
|
||||
terminal=terminal and index == len(stages) - 1,
|
||||
next_kind=stages[index + 1][0].kind
|
||||
if declaration.mixer_exit and index + 1 < len(stages)
|
||||
else declaration.next_kind,
|
||||
)
|
||||
declarations.append(declaration)
|
||||
bindings.append((norm, item[2] if len(item) == 3 else {}))
|
||||
boundaries = []
|
||||
for index, (declaration, (norm, options)) in enumerate(zip(declarations, bindings)):
|
||||
build = (
|
||||
make_attn_stage
|
||||
if declaration.kind is StageKind.ATTENTION
|
||||
else make_ffn_stage
|
||||
)
|
||||
boundaries.append(
|
||||
build(
|
||||
declaration=declaration,
|
||||
norm=norm,
|
||||
following=declarations[index + 1]
|
||||
if index + 1 < len(declarations)
|
||||
else None,
|
||||
**options,
|
||||
)
|
||||
)
|
||||
for producer, consumer in zip(boundaries, boundaries[1:]):
|
||||
if (
|
||||
producer.kind is not StageKind.ATTENTION
|
||||
or producer.declaration.mixer_exit
|
||||
or consumer.kind is not StageKind.FFN
|
||||
):
|
||||
continue
|
||||
for variant, steps in producer.plan._paths.items():
|
||||
predicate = consumer.plan._paths[variant].ffn.preserves_residual
|
||||
if predicate is not None:
|
||||
producer.plan._paths[variant] = msgspec.structs.replace(
|
||||
steps,
|
||||
attention=msgspec.structs.replace(
|
||||
steps.attention, capture_preserves_residual=predicate
|
||||
),
|
||||
)
|
||||
producer.plan._steps = producer.plan._paths[BatchVariant.ORDINARY]
|
||||
producer.plan._cp_steps = producer.plan._paths.get(
|
||||
BatchVariant.CONTEXT_PARALLEL
|
||||
)
|
||||
producer.plan._input_scattered_steps = producer.plan._paths.get(
|
||||
BatchVariant.INPUT_SCATTERED
|
||||
)
|
||||
producer.plan._sp_steps = producer.plan._paths.get(
|
||||
BatchVariant.SEQUENCE_PARALLEL
|
||||
)
|
||||
return tuple(boundaries)
|
||||
@@ -0,0 +1,148 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""Select fused completion and read kernels for each consumer."""
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.communicator.boundary import FusedMlpInput
|
||||
from sglang.srt.layers.communicator.layout import SumGroup
|
||||
from sglang.srt.layers.communicator.output import UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
apply_aiter_all_reduce_fusion,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.moe import post_experts_reduction_group
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
|
||||
|
||||
def attention_fusions(plan) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete what the previous layer left together
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly; the boundary tries them only when the update it writes in is a
|
||||
plain add. A backend's come first."""
|
||||
given = plan.fusions.attention_input(plan) if plan.fusions else ()
|
||||
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
|
||||
return given
|
||||
plan._attn_input_fuses_quant = (
|
||||
plan.enable_fused_ar_quant
|
||||
and _use_aiter
|
||||
and hasattr(plan.norm, "forward_with_allreduce_fusion_quant_per_group")
|
||||
)
|
||||
return (*given, partial(complete_attention_input, plan))
|
||||
|
||||
|
||||
def complete_attention_input(
|
||||
plan,
|
||||
owed: UnreducedOutput,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
post_residual_addition: Optional[torch.Tensor],
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Complete the sum the previous layer left, add it to the residual and
|
||||
apply the input norm in one aiter or flashinfer kernel; None when the
|
||||
kernel does not take this batch. The result is not quantized for
|
||||
``quant_format``."""
|
||||
if (
|
||||
not isinstance(owed, UnreducedOutput)
|
||||
# The kernel does not add it.
|
||||
or post_residual_addition is not None
|
||||
# The kernel reduces over the MoE output's group.
|
||||
or owed.group is not post_experts_reduction_group()
|
||||
):
|
||||
return None
|
||||
hidden_states = owed.partial
|
||||
if not (
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
):
|
||||
return None
|
||||
if plan._attn_input_fuses_quant:
|
||||
# Falls back to AR+RMSNorm + separate quant internally when the
|
||||
# fully-fused kernel cannot service the shape.
|
||||
quant_result = plan.norm.forward_with_allreduce_fusion_quant_per_group(
|
||||
hidden_states,
|
||||
residual,
|
||||
use_attn_tp_group=False,
|
||||
keep_bf16=plan.fused_ar_quant_keep_bf16,
|
||||
)
|
||||
if quant_result is not None:
|
||||
return quant_result
|
||||
return plan.norm.forward_with_allreduce_fusion(
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
|
||||
|
||||
def ffn_fusions(plan) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried. They add the residual plainly; the boundary tries
|
||||
them only when the update it writes in is a plain add. A backend's come
|
||||
first."""
|
||||
given = plan.fusions.ffn_input(plan) if plan.fusions else ()
|
||||
if not hasattr(plan.norm, "forward_with_allreduce_fusion"):
|
||||
return given
|
||||
return (
|
||||
*given,
|
||||
FusedMlpInput(
|
||||
completes=SumGroup.ATTN_TP,
|
||||
run=partial(complete_ffn_input, plan),
|
||||
preserves_residual=(
|
||||
flashinfer_preserves_residual
|
||||
if isinstance(plan.norm, (RMSNorm, GemmaRMSNorm))
|
||||
and type(plan.norm).forward_with_allreduce_fusion
|
||||
in (
|
||||
RMSNorm.forward_with_allreduce_fusion,
|
||||
GemmaRMSNorm.forward_with_allreduce_fusion,
|
||||
)
|
||||
else None
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_preserves_residual(value, forward_batch):
|
||||
"""The CUDA wrapper returns a fresh residual, including backend fallback."""
|
||||
return (
|
||||
not _use_aiter
|
||||
and get_parallel().attn_tp_size > 1
|
||||
and apply_flashinfer_allreduce_fusion(value.shape[0])
|
||||
)
|
||||
|
||||
|
||||
def complete_ffn_input(
|
||||
plan,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""The attention-TP all-reduce, residual add and norm in one aiter or
|
||||
flashinfer kernel; None when neither takes the batch."""
|
||||
if not (
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
):
|
||||
return None
|
||||
return plan.norm.forward_with_allreduce_fusion(
|
||||
hidden_states, residual, use_attn_tp_group=True
|
||||
)
|
||||
@@ -42,8 +42,8 @@ from sglang.srt.layers.communicator.boundary import (
|
||||
sequence_parallel_layer_sides,
|
||||
with_residual,
|
||||
)
|
||||
from sglang.srt.layers.communicator.exit import _reduce_and_redistribute_output_step
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -56,6 +56,7 @@ from sglang.srt.layers.communicator.layout import (
|
||||
sparse_moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.communicator.legacy_stage import StageCommunicator
|
||||
from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
_all_reduce_then_to_local_tokens,
|
||||
@@ -63,7 +64,6 @@ from sglang.srt.layers.communicator.ops import (
|
||||
_hand_scattered_input_to_attention,
|
||||
_mlp_input_without_dp,
|
||||
_redistribute_output,
|
||||
_reduce_and_redistribute_output_step,
|
||||
_to_local_tokens,
|
||||
move_rows,
|
||||
)
|
||||
@@ -189,79 +189,6 @@ def _can_defer_ffn_reduction(forward_batch: ForwardBatch, boundary=None) -> bool
|
||||
)
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
|
||||
self._layer = layer
|
||||
self._name = name
|
||||
self._norm_name = norm_name
|
||||
|
||||
@property
|
||||
def norm(self) -> torch.nn.Module:
|
||||
"""The norm the stage reads its input with, one of the layer's."""
|
||||
return getattr(self._layer, self._norm_name)
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
entry = getattr(steps, self._name)
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self._name} stage"
|
||||
)
|
||||
return entry
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
context = self._layer._context
|
||||
stream = residual_batch.current(forward_batch)
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
):
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
call["pending"] = stream.pending
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, context, **call
|
||||
)
|
||||
if entry.input_move is not None:
|
||||
hidden_states = entry.input_move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=context,
|
||||
)
|
||||
if entry.handoff is not None:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self._layer.qkv_latent_func
|
||||
)
|
||||
stream.write(residual)
|
||||
return hidden_states, stream
|
||||
|
||||
|
||||
class LayerFusions(Protocol):
|
||||
"""The fused kernels a fusion backend gives a layer, bound to it and tried
|
||||
before the layer's own: at its attention input (each takes what the previous
|
||||
@@ -336,10 +263,6 @@ class LayerCommunicator:
|
||||
self._residual = residual
|
||||
self.fusions = fusions
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
force_layernorm_before_dp_gather
|
||||
)
|
||||
# The fused kernels every batch's attention input tries first.
|
||||
self._attn_input_fusions = self._select_attn_input_fusions()
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
@@ -799,7 +722,6 @@ class LayerCommunicator:
|
||||
else move(
|
||||
hidden_states=residual_value,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
keeps = self.attn.entry(forward_batch).capture_preserves_residual
|
||||
@@ -850,7 +772,7 @@ class LayerCommunicator:
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
return self.attn.prepare(
|
||||
return self.attn._prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -979,8 +901,6 @@ class LayerCommunicator:
|
||||
):
|
||||
steps = self._batch_steps(forward_batch)
|
||||
self.publish_mlp_lora_layout(steps)
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
if isinstance(residual, ResidualStream) and self.stage_edges is None:
|
||||
# The paired boundary records the attention contribution once.
|
||||
@@ -989,12 +909,13 @@ class LayerCommunicator:
|
||||
self._residual.attention_update,
|
||||
declared_sum=steps.ffn.input_sum,
|
||||
)
|
||||
return self.ffn.prepare(
|
||||
return self.ffn._prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
steps,
|
||||
update=self._residual.attention_update,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
def maybe_prefetch_next_full_attention_kv(
|
||||
@@ -1180,7 +1101,6 @@ class LayerCommunicator:
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_facts.is_layer_sparse,
|
||||
)
|
||||
@@ -1317,7 +1237,7 @@ class LayerCommunicator:
|
||||
input when no fused kernel takes it: when the next layer would run the
|
||||
same all-reduce the FFN itself would have."""
|
||||
return (
|
||||
self._context.tp_size > 1
|
||||
get_parallel().tp_size > 1
|
||||
and not self.is_last_layer
|
||||
and self._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
and _can_defer_ffn_reduction(forward_batch, self)
|
||||
@@ -1361,15 +1281,23 @@ class MixerExit:
|
||||
its input), and when its output declaration permits deferring the sum. The
|
||||
consumer decides whether to fuse its completion with the input norm."""
|
||||
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update", "_declared_sum")
|
||||
__slots__ = (
|
||||
"skips_reduction",
|
||||
"_hands_on",
|
||||
"_scope",
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
produced = communicator._batch_steps(forward_batch).ffn_output
|
||||
self._stream = None
|
||||
self._update = produced.update
|
||||
self._declared_sum = produced.group if produced.always_leaves else None
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and communicator._context.tp_size > 1
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and communicator._ffn_sum_can_move_to_next_layer(forward_batch)
|
||||
)
|
||||
@@ -1386,14 +1314,17 @@ class MixerExit:
|
||||
def finish(self, hidden_states: torch.Tensor, residual=None):
|
||||
"""The mixer's output: its partial sum to the FFN stage after it, as a
|
||||
value that owes the sum to a mixer after it, or complete."""
|
||||
if self._stream is not None:
|
||||
residual = self._stream
|
||||
if self._hands_on:
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.leave(
|
||||
output = residual.leave(
|
||||
hidden_states, self._update, declared_sum=self._declared_sum
|
||||
), residual
|
||||
)
|
||||
return output if self._stream is not None else (output, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1412,9 +1343,11 @@ class FfnExit:
|
||||
"_update",
|
||||
"_declared_sum",
|
||||
"_scope",
|
||||
"_stream",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
self._stream = None
|
||||
self.communicator = communicator
|
||||
self.forward_batch = forward_batch
|
||||
completion = communicator._select_ffn_completion(forward_batch)
|
||||
@@ -1441,7 +1374,7 @@ class FfnExit:
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor = None
|
||||
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input, or postprocess."""
|
||||
stream = residual_batch.current(self.forward_batch)
|
||||
@@ -1452,9 +1385,10 @@ class FfnExit:
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
else:
|
||||
hidden_states, residual = self._complete(hidden_states, residual)
|
||||
return self.communicator._leave_ffn_output(
|
||||
result = self.communicator._leave_ffn_output(
|
||||
hidden_states, residual, stream, self._update, self._declared_sum
|
||||
)
|
||||
return result[0] if self._stream is not None else result
|
||||
|
||||
|
||||
class MHCLayerCommunicator(LayerCommunicator):
|
||||
@@ -1501,7 +1435,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
residual=self.mhc.layer_residual(),
|
||||
)
|
||||
# MHC has not been run with input-scattered attention under attention CP.
|
||||
if self._context.attn_cp_size > 1 and self._input_can_be_scattered():
|
||||
if get_parallel().attn_cp_size > 1 and self._input_can_be_scattered():
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator with input-scattered attention under attention CP"
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
"""Token axes, layouts and process groups of what a layer hands across its
|
||||
boundaries, and the configuration and batch facts they depend on."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import Dict, FrozenSet, List, Mapping, Optional
|
||||
|
||||
@@ -94,38 +93,6 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
|
||||
return _batch_size(forward_batch) > 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
attn_tp_rank: int
|
||||
attn_tp_size: int
|
||||
attn_dp_size: int
|
||||
attn_cp_rank: int
|
||||
attn_cp_size: int
|
||||
tp_size: int
|
||||
cache = None
|
||||
tp_rank: int
|
||||
force_layernorm_before_dp_gather: bool = False
|
||||
|
||||
@classmethod
|
||||
def init_new(cls):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_cp_size = get_parallel().attn_cp_size
|
||||
attn_cp_rank = get_parallel().attn_cp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
return cls(
|
||||
attn_tp_rank=attn_tp_rank,
|
||||
attn_tp_size=attn_tp_size,
|
||||
attn_dp_size=attn_dp_size,
|
||||
attn_cp_rank=attn_cp_rank,
|
||||
attn_cp_size=attn_cp_size,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
|
||||
|
||||
def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]:
|
||||
"""The real rows each MoE-CP rank contributes when this batch's FFN input is
|
||||
gathered across the MoE-CP group, or None when it is not: only a context
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A stage's model-facing input and output boundary."""
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from sglang.srt.layers.communicator.boundary import BoundarySteps, StageEntry
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD
|
||||
from sglang.srt.layers.communicator.residual.batch import current
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.layer import LayerCommunicator
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
def __init__(self, layer: "LayerCommunicator", name: str, norm_name: str):
|
||||
self._layer = layer
|
||||
self._name = name
|
||||
self.norm = getattr(layer, norm_name)
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
entry = getattr(steps, self._name)
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self._name} stage"
|
||||
)
|
||||
return entry
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
stream = current(forward_batch)
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
if (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
):
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
call["pending"] = stream.pending
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, **call
|
||||
)
|
||||
if entry.input_move is not None:
|
||||
hidden_states = entry.input_move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if entry.handoff is not None:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self._layer.qkv_latent_func
|
||||
)
|
||||
stream.write(residual)
|
||||
return hidden_states, stream
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
return self._layer.input_rows
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
return self._layer.input_on_attention_tp_slices
|
||||
|
||||
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
|
||||
stream = current(forward_batch)
|
||||
if self._name == "attention":
|
||||
hidden_states, stream = (
|
||||
self._layer.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states, stream, forward_batch, **call
|
||||
)
|
||||
)
|
||||
else:
|
||||
steps = self._layer._batch_steps(forward_batch)
|
||||
self._layer.publish_mlp_lora_layout(steps)
|
||||
if cache is not None:
|
||||
call["cache"] = cache
|
||||
hidden_states, stream = self._prepare(
|
||||
hidden_states, stream, forward_batch, steps, **call
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
"""Hand this attention's actual contribution to the following stage."""
|
||||
if self._name != "attention" or self._layer.stage_edges is not None:
|
||||
raise RuntimeError(
|
||||
"use the stage exit scope for an FFN or single-stage mixer"
|
||||
)
|
||||
return current(forward_batch).leave(
|
||||
hidden_states,
|
||||
self._layer._residual.attention_update,
|
||||
declared_sum=self._layer._batch_steps(forward_batch).ffn.input_sum,
|
||||
)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
result = (
|
||||
self._layer.ffn_exit(forward_batch)
|
||||
if self._name == "ffn"
|
||||
else self._layer.mixer_exit(forward_batch)
|
||||
)
|
||||
result._stream = current(forward_batch)
|
||||
return result
|
||||
|
||||
def from_pp(self, tensors, forward_batch, **kwargs):
|
||||
hidden_states, stream = self._layer.from_pp(tensors, forward_batch, **kwargs)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def snapshot(self, hidden_states, forward_batch, *, at_input=False):
|
||||
return self._layer.snapshot(
|
||||
hidden_states, current(forward_batch), at_input=at_input
|
||||
)
|
||||
|
||||
def capture_output(self, hidden_states, forward_batch, **kwargs):
|
||||
return self._layer.capture_output(
|
||||
hidden_states, current(forward_batch), **kwargs
|
||||
)
|
||||
|
||||
def postprocess(self, hidden_states, forward_batch):
|
||||
hidden_states, stream = self._layer.postprocess_layer(
|
||||
hidden_states, current(forward_batch), forward_batch
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
@@ -38,7 +38,6 @@ from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -56,7 +55,6 @@ from sglang.srt.layers.communicator.residual.stream import DeclaredSum
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
can_use_dp_reduce_scatter,
|
||||
dp_gather_partial,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
@@ -69,7 +67,6 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
moe_cp_all_gather_into_tensor,
|
||||
)
|
||||
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
@@ -86,28 +83,30 @@ if _is_npu:
|
||||
def tp_reduce_scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
context: "CommunicateContext",
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Module-level so MHC communicators can reuse it without a
|
||||
``LayerCommunicator`` instance."""
|
||||
"""Complete a TP sum onto each rank's token slice."""
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
assert hidden_states.shape[0] % context.tp_size == 0, (
|
||||
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
|
||||
assert hidden_states.shape[0] % parallel.tp_size == 0, (
|
||||
f"Expected total tokens {hidden_states.shape[0]} % tp_size {parallel.tp_size} to be 0"
|
||||
)
|
||||
local_tokens = hidden_states.shape[0] // context.tp_size
|
||||
local_tokens = hidden_states.shape[0] // parallel.tp_size
|
||||
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
|
||||
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
|
||||
parallel.tp_group.reduce_scatter_tensor(output, hidden_states)
|
||||
if residual is not None:
|
||||
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
||||
residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
return output, residual
|
||||
|
||||
|
||||
def tp_slice(hidden_states, residual, context):
|
||||
def tp_slice(hidden_states, residual):
|
||||
"""The rows reduce-scatter would return, when the sum is already complete."""
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank].clone()
|
||||
parallel = get_parallel()
|
||||
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
|
||||
parallel.tp_rank
|
||||
].clone()
|
||||
if residual is not None:
|
||||
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
||||
residual = residual.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -116,7 +115,6 @@ class CommunicateSimpleFn:
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
@@ -124,18 +122,18 @@ class CommunicateSimpleFn:
|
||||
def _scattered_to_tp_attn_full(
|
||||
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
parallel = get_parallel()
|
||||
if isinstance(hidden_states, tuple):
|
||||
gathered_hidden_states = []
|
||||
for local_hidden_states in hidden_states:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
parallel.tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
output = torch.empty(
|
||||
(
|
||||
local_hidden_states.shape[0] * context.attn_tp_size,
|
||||
local_hidden_states.shape[0] * parallel.attn_tp_size,
|
||||
*local_hidden_states.shape[1:],
|
||||
),
|
||||
dtype=local_hidden_states.dtype,
|
||||
@@ -152,10 +150,11 @@ class CommunicateSimpleFn:
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states: torch.Tensor, context: CommunicateContext
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
parallel = get_parallel()
|
||||
local_hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[
|
||||
parallel.attn_tp_rank
|
||||
]
|
||||
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
|
||||
return local_hidden_states
|
||||
@@ -250,8 +249,8 @@ def _mlp_input_without_dp(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
@@ -274,8 +273,8 @@ def _mlp_input_without_dp(
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
if _is_npu and cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, cache)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
@@ -284,8 +283,8 @@ def _mlp_input_dp_replicate(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
@@ -324,8 +323,8 @@ def _mlp_input_dp_partial(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
@@ -336,7 +335,7 @@ def _mlp_input_dp_partial(
|
||||
shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
if context.attn_tp_rank == 0:
|
||||
if get_parallel().attn_tp_rank == 0:
|
||||
hidden_states += residual
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
hidden_states = _reduce_and_redistribute_output_to_dp(
|
||||
@@ -395,7 +394,7 @@ def _tp_all_gather_scattered_rows(
|
||||
|
||||
|
||||
def _dispatch_consumer(
|
||||
hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call
|
||||
hidden_states, residual, forward_batch, norm, *, paths, update=ADD, **call
|
||||
):
|
||||
if residual is None:
|
||||
# Enter/read-only paths do not execute an update.
|
||||
@@ -405,9 +404,7 @@ def _dispatch_consumer(
|
||||
prepare = paths[update.adds_plainly]
|
||||
except KeyError:
|
||||
raise RuntimeError("producer update has no bound input path") from None
|
||||
return prepare(
|
||||
hidden_states, residual, forward_batch, norm, context, update=update, **call
|
||||
)
|
||||
return prepare(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _consumer_step(
|
||||
@@ -415,7 +412,6 @@ def _consumer_step(
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
step: Callable,
|
||||
adds_plainly: bool,
|
||||
@@ -473,9 +469,7 @@ def _consumer_step(
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
return step(
|
||||
hidden_states, residual, forward_batch, norm, context, update=update, **call
|
||||
)
|
||||
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
|
||||
|
||||
|
||||
def _read_input(
|
||||
@@ -483,8 +477,8 @@ def _read_input(
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
layer_input: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
@@ -498,7 +492,7 @@ def _read_input(
|
||||
starts its residual from its input."""
|
||||
enters = residual is None and enters_stack
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(hidden_states, residual, context)
|
||||
hidden_states, residual = layer_input(hidden_states, residual)
|
||||
if enters:
|
||||
hidden_states, residual = read.enter(hidden_states), None
|
||||
if residual is None:
|
||||
@@ -519,17 +513,15 @@ def _mlp_input_scatter(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states, context
|
||||
)
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(hidden_states)
|
||||
if scatters_residual:
|
||||
residual = update.residual_to_attn_tp_shard(residual, context)
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
@@ -538,15 +530,15 @@ def _mlp_input_slice(
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
scatters_residual: bool,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone()
|
||||
hidden_states = _redistribute_to_attn_tp_shards(hidden_states).clone()
|
||||
if scatters_residual and residual is not None:
|
||||
residual = update.residual_to_attn_tp_shard(residual, context)
|
||||
residual = update.residual_to_attn_tp_shard(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
@@ -555,8 +547,8 @@ def _mlp_input_on_residual_shard(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
reduces: bool = True,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
@@ -565,11 +557,12 @@ def _mlp_input_on_residual_shard(
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
shard = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
if reduces:
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
parallel.tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
else:
|
||||
shard = shard.clone()
|
||||
shard, residual = read.update_and_read(update, shard, residual, layernorm)
|
||||
@@ -582,13 +575,13 @@ def _mlp_input_residual_into_sum(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context, read
|
||||
hidden_states, residual, layernorm, read
|
||||
)
|
||||
|
||||
|
||||
@@ -596,13 +589,13 @@ def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
read: StageRead = NORM_READ,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
scattered_states = hidden_states.tensor_split(parallel.tp_size)[parallel.tp_rank]
|
||||
scattered_states += residual
|
||||
residual = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return read.read(residual, layernorm)
|
||||
@@ -613,8 +606,8 @@ def _mlp_input_gather_attention_cp(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
@@ -622,7 +615,7 @@ def _mlp_input_gather_attention_cp(
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context, update=update
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
return dsa_cp_gather_hidden_states(hidden_states), residual
|
||||
|
||||
@@ -632,8 +625,8 @@ def _mlp_input_gather_moe_cp(
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
@@ -649,7 +642,7 @@ def _mlp_input_gather_moe_cp(
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context, update=update
|
||||
hidden_states, residual, forward_batch, layernorm, update=update, cache=cache
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
@@ -709,27 +702,6 @@ def _redistribute_output_from_moe_cp(
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _reduce_and_redistribute_output_step(
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings an FFN output gathered over attention
|
||||
DP back to this rank's tokens when the FFN leaves its sum to it (see
|
||||
StageOutput); None when the FFN reduces the output and only a scatter
|
||||
remains."""
|
||||
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
|
||||
return _reduce_and_redistribute_output_varlen
|
||||
if (
|
||||
leaves_for_reduce_scatter
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and can_use_dp_reduce_scatter()
|
||||
):
|
||||
return _reduce_and_redistribute_output_max_len
|
||||
return None
|
||||
|
||||
|
||||
def _to_local_tokens(
|
||||
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -756,38 +728,15 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
is_layer_sparse: bool = False,
|
||||
):
|
||||
step = (
|
||||
_reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=allow_reduce_scatter,
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever it
|
||||
# applies (should_skip_post_experts_all_reduce).
|
||||
leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
|
||||
)
|
||||
or _redistribute_output
|
||||
)
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
|
||||
@staticmethod
|
||||
def _gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
update: StageUpdate = ADD,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -799,7 +748,6 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
@@ -810,17 +758,20 @@ class CommunicateSummableTensorPairFn:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
parallel = get_parallel()
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
hidden_states = hidden_states.tensor_split(parallel.tp_size)[
|
||||
parallel.tp_rank
|
||||
]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = update.update(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
local_states.shape[0] * parallel.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
parallel.tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
@@ -828,17 +779,15 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
assert residual is None, "not yet handled residual!=None"
|
||||
return _redistribute_to_attn_tp_shards(hidden_states, context), None
|
||||
return _redistribute_to_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _take_back_attention_cp_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: this rank's shard of a complete output gathered in
|
||||
@@ -852,7 +801,6 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""DSA and MLA CP: sum the FFN output over the attention-CP group and
|
||||
@@ -864,7 +812,6 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""This rank's CP shard of the rows the DP gather put in its DP group's
|
||||
@@ -886,7 +833,6 @@ class CommunicateSummableTensorPairFn:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
**kwargs,
|
||||
):
|
||||
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
||||
@@ -905,7 +851,7 @@ class CommunicateSummableTensorPairFn:
|
||||
if rows is not None:
|
||||
hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows)
|
||||
|
||||
if context.attn_dp_size > 1:
|
||||
if get_parallel().attn_dp_size > 1:
|
||||
hidden_states = _to_local_tokens(
|
||||
_redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
|
||||
@@ -21,6 +21,18 @@ import torch
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
|
||||
|
||||
class OutputTransform(msgspec.Struct, frozen=True):
|
||||
"""An operation on the producer contribution, before residual addition.
|
||||
|
||||
Some existing implementations apply it to a partial before reduce-scatter
|
||||
but to a complete value after all-reduce. These are explicit alternatives,
|
||||
not an assumption that an arbitrary operation commutes with reduction.
|
||||
"""
|
||||
|
||||
apply: Callable[[torch.Tensor], torch.Tensor]
|
||||
before_reduce_scatter: bool = False
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its sum, left for the next layer's input.
|
||||
Hand it to the next layer, or pass it through reduce_output() before reading
|
||||
|
||||
@@ -38,7 +38,7 @@ class StageUpdate(Protocol):
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the output written into it."""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
|
||||
def residual_to_attn_tp_shard(self, residual) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual, with whatever the
|
||||
update reads along with it."""
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ def finish_layer_stack(
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
|
||||
def norm_output(hidden_states, residual, norm, capture_output=None):
|
||||
def norm_output(hidden_states, residual, norm, capture_output=None, **read_kwargs):
|
||||
"""The final norm and an optional capture of the same updated residual.
|
||||
|
||||
Keep add and norm together: normalizing a separately rounded residual is
|
||||
@@ -116,7 +116,7 @@ def norm_output(hidden_states, residual, norm, capture_output=None):
|
||||
if capture_output is not None:
|
||||
capture_output(hidden_states)
|
||||
return norm(hidden_states)
|
||||
hidden_states, residual = norm(hidden_states, residual)
|
||||
hidden_states, residual = norm(hidden_states, residual, **read_kwargs)
|
||||
if capture_output is not None:
|
||||
capture_output(residual)
|
||||
return hidden_states
|
||||
|
||||
@@ -273,8 +273,8 @@ class Add:
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return _redistribute_to_attn_tp_shards(residual, context)
|
||||
def residual_to_attn_tp_shard(self, residual):
|
||||
return _redistribute_to_attn_tp_shards(residual)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
@@ -1,18 +1,82 @@
|
||||
"""Own one invocation's residual stream, including independent TBO children."""
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Residual access for one layer-stack invocation or TBO microbatch."""
|
||||
|
||||
from sglang.srt.layers.communicator.residual import access
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
|
||||
def start(forward_batch):
|
||||
"""Begin a fresh embedding-side invocation and return its stream."""
|
||||
stream = ResidualStream()
|
||||
forward_batch.residual_stream = stream
|
||||
return stream
|
||||
|
||||
|
||||
def current(forward_batch):
|
||||
"""Return the invocation's authoritative stream while its stack is active."""
|
||||
stream = forward_batch.residual_stream
|
||||
if stream is None:
|
||||
raise RuntimeError("start the layer stack before entering a stage")
|
||||
return stream
|
||||
|
||||
|
||||
def finish(hidden_states, forward_batch):
|
||||
return current(forward_batch).complete(hidden_states)
|
||||
|
||||
|
||||
def take_output(hidden_states, forward_batch):
|
||||
"""Release a terminal output whose residual update has already been written."""
|
||||
stream = current(forward_batch)
|
||||
stream.check(hidden_states)
|
||||
if stream.pending is not None:
|
||||
raise RuntimeError("write the residual update before taking the final output")
|
||||
forward_batch.residual_stream = None
|
||||
return hidden_states
|
||||
|
||||
|
||||
def norm(hidden_states, forward_batch, layernorm, capture_output=None, **read_kwargs):
|
||||
hidden_states, residual = current(forward_batch).finish(hidden_states)
|
||||
# The terminal consumer now owns the pair. Do not keep layer buffers alive
|
||||
# through logits processing or the next forward on this batch.
|
||||
forward_batch.residual_stream = None
|
||||
return access.norm_output(
|
||||
hidden_states, residual, layernorm, capture_output, **read_kwargs
|
||||
)
|
||||
|
||||
|
||||
def to_pp(hidden_states, forward_batch, *, preserve_declared=True):
|
||||
hidden_states, residual = current(forward_batch).finish(
|
||||
hidden_states, preserve_declared=preserve_declared
|
||||
)
|
||||
forward_batch.residual_stream = None
|
||||
return PPProxyTensors({"hidden_states": hidden_states, "residual": residual})
|
||||
|
||||
|
||||
def snapshot(hidden_states, forward_batch):
|
||||
return current(forward_batch).snapshot(hidden_states)
|
||||
|
||||
|
||||
def add_to_output(hidden_states, forward_batch, extra):
|
||||
output, _ = access.add_to_output(hidden_states, current(forward_batch), extra)
|
||||
return output
|
||||
|
||||
|
||||
def fold(hidden_states, forward_batch):
|
||||
output, _ = access.fold(hidden_states, current(forward_batch))
|
||||
return output
|
||||
|
||||
|
||||
def written(hidden_states, forward_batch):
|
||||
forward_batch.residual_stream = ResidualStream(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
@@ -20,6 +20,7 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -96,8 +97,9 @@ class MHCState:
|
||||
self.h_res = None
|
||||
self.h_post = None
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
rank, size = context.attn_tp_rank, context.attn_tp_size
|
||||
def residual_to_attn_tp_shard(self, residual):
|
||||
parallel = get_parallel()
|
||||
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
self.h_res = self.h_res.tensor_split(size)[rank]
|
||||
self.h_post = self.h_post.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
@@ -151,8 +153,8 @@ class _AttentionUpdate:
|
||||
def update(self, hidden_states, residual):
|
||||
return self.state.mlp_combine(hidden_states, residual)
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return self.state.residual_to_attn_tp_shard(residual, context)
|
||||
def residual_to_attn_tp_shard(self, residual):
|
||||
return self.state.residual_to_attn_tp_shard(residual)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return self.state.residual_from_attn_tp_shards(residual)
|
||||
@@ -198,8 +200,8 @@ class _FfnUpdate:
|
||||
hidden_states = hc_contract(hidden_states, self.state.hc_mult)
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return self.state.residual_to_attn_tp_shard(residual, context)
|
||||
def residual_to_attn_tp_shard(self, residual):
|
||||
return self.state.residual_to_attn_tp_shard(residual)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return self.state.residual_from_attn_tp_shards(residual)
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A stage's model-facing input and output boundary."""
|
||||
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.communicator.adapters import branch
|
||||
from sglang.srt.layers.communicator.adapters.lora import (
|
||||
publish_attention,
|
||||
publish_ffn,
|
||||
)
|
||||
from sglang.srt.layers.communicator.boundary import (
|
||||
BoundarySteps,
|
||||
StageEntry,
|
||||
StageKind,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import CommunicateSimpleFn
|
||||
from sglang.srt.layers.communicator.residual.access import buffer, from_pp
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
ADD,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.batch import current
|
||||
from sglang.srt.layers.communicator.residual.stream import DeclaredSum, ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.communicator.construction import StagePlan
|
||||
from sglang.srt.layers.communicator.factories import StageDeclaration
|
||||
|
||||
|
||||
class StageCommunicator:
|
||||
"""One stage of a layer, its attention or its FFN: the boundary into it,
|
||||
run with the stage's norm on the steps the layer chose for the batch."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
plan: "StagePlan",
|
||||
kind: StageKind,
|
||||
norm,
|
||||
*,
|
||||
declaration: Optional["StageDeclaration"] = None,
|
||||
):
|
||||
self.plan = plan
|
||||
self.kind = kind
|
||||
self.norm = norm
|
||||
self._declaration = declaration
|
||||
|
||||
@property
|
||||
def declaration(self) -> "StageDeclaration":
|
||||
"""Immutable source contract for constructing another local sequence."""
|
||||
if self._declaration is None:
|
||||
raise RuntimeError("this boundary was constructed from raw layout edges")
|
||||
return self._declaration
|
||||
|
||||
def entry(
|
||||
self, forward_batch: ForwardBatch, steps: Optional[BoundarySteps] = None
|
||||
) -> StageEntry:
|
||||
if steps is None:
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
entry = steps.attention if self.kind is StageKind.ATTENTION else steps.ffn
|
||||
if entry is None:
|
||||
raise NotImplementedError(
|
||||
f"a layer that is one stage has no {self.kind.name.lower()} stage"
|
||||
)
|
||||
return entry
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch: ForwardBatch,
|
||||
steps: Optional[BoundarySteps] = None,
|
||||
**call,
|
||||
):
|
||||
"""The stage's input and the residual, from the previous stage's output
|
||||
(``call``: what the stage's read takes, e.g. the attention's
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
stream = (
|
||||
residual
|
||||
if isinstance(residual, ResidualStream)
|
||||
else ResidualStream(residual)
|
||||
)
|
||||
if not isinstance(residual, ResidualStream):
|
||||
if residual is not None or entry.input_sum is not None:
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, call.get("update", ADD), declared_sum=entry.input_sum
|
||||
)
|
||||
elif (
|
||||
stream.pending is None
|
||||
and stream.residual is None
|
||||
and entry.input_sum is not None
|
||||
):
|
||||
hidden_states = stream.leave(
|
||||
hidden_states, ADD, declared_sum=entry.input_sum
|
||||
)
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
call["pending"] = stream.pending
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, **call
|
||||
)
|
||||
if entry.input_move is not None:
|
||||
hidden_states = entry.input_move(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if entry.handoff is not None:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self.plan.qkv_latent_func
|
||||
)
|
||||
stream.write(residual)
|
||||
return hidden_states, stream
|
||||
|
||||
@property
|
||||
def fusions(self):
|
||||
return self.plan.fusions
|
||||
|
||||
@property
|
||||
def input_rows(self):
|
||||
return self.plan.input_rows
|
||||
|
||||
@property
|
||||
def input_on_attention_tp_slices(self):
|
||||
return self.plan.input_on_attention_tp_slices
|
||||
|
||||
def prepare(self, hidden_states, forward_batch, *, cache=None, **call):
|
||||
if cache is not None:
|
||||
call["cache"] = cache
|
||||
prepare = (
|
||||
self._prepare_attention
|
||||
if self.kind is StageKind.ATTENTION
|
||||
else self._prepare_input
|
||||
)
|
||||
hidden_states, forward_batch.residual_stream = prepare(
|
||||
hidden_states, current(forward_batch), forward_batch, **call
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def finish(self, hidden_states, forward_batch):
|
||||
"""Hand this attention's actual contribution to the following stage."""
|
||||
if self.kind is not StageKind.ATTENTION or not self.plan.fixed_output:
|
||||
raise RuntimeError(
|
||||
"use the stage exit scope for an FFN or single-stage mixer"
|
||||
)
|
||||
produced = self.plan.produced(self.kind, forward_batch)
|
||||
return current(forward_batch).leave(
|
||||
hidden_states,
|
||||
produced.update,
|
||||
declared_sum=produced.group if produced.always_leaves else None,
|
||||
)
|
||||
|
||||
def exit(self, forward_batch):
|
||||
result = (
|
||||
self.plan.output.ffn_exit(forward_batch)
|
||||
if self.kind is StageKind.FFN
|
||||
else self.plan.output.mixer_exit(forward_batch)
|
||||
)
|
||||
result._stream = current(forward_batch)
|
||||
return result
|
||||
|
||||
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
|
||||
hidden_states, residual = from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=self.plan.residual_in_hidden,
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
declared_sum = None
|
||||
if forward_batch is not None:
|
||||
steps = self.plan._batch_steps(forward_batch)
|
||||
entry = steps.attention or steps.ffn
|
||||
declared_sum = entry.input_sum
|
||||
hidden_states, forward_batch.residual_stream = ResidualStream.arrive(
|
||||
hidden_states, residual, ADD, declared_sum=declared_sum
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def snapshot(self, hidden_states, forward_batch):
|
||||
return current(forward_batch).snapshot(hidden_states)
|
||||
|
||||
def capture_output(self, hidden_states, forward_batch, *, skip_empty=False):
|
||||
stream = current(forward_batch)
|
||||
if skip_empty:
|
||||
storage = buffer(hidden_states)
|
||||
if storage is not None and storage.shape[0] == 0:
|
||||
return hidden_states, None
|
||||
if stream.pending is None or not isinstance(stream.pending.owed, DeclaredSum):
|
||||
hidden_states = stream.complete(hidden_states)
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def postprocess(self, hidden_states, forward_batch):
|
||||
hidden_states, stream = self.plan.output.postprocess_layer(
|
||||
hidden_states, current(forward_batch), forward_batch
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def branch_input(self, source, hidden_states, forward_batch):
|
||||
hidden_states, stream = branch.branch_input(
|
||||
self.plan,
|
||||
source.plan,
|
||||
hidden_states,
|
||||
current(forward_batch),
|
||||
forward_batch,
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def branch_output(self, hidden_states, forward_batch):
|
||||
return branch.branch_output(self.plan, hidden_states, forward_batch)
|
||||
|
||||
def merge_branch(self, contribution, hidden_states, source, forward_batch):
|
||||
hidden_states, stream = branch.merge_branch(
|
||||
self.plan,
|
||||
contribution,
|
||||
hidden_states,
|
||||
current(forward_batch),
|
||||
source.plan,
|
||||
forward_batch,
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states
|
||||
|
||||
def _prepare_input(self, hidden_states, residual, forward_batch, **call):
|
||||
plan = self.plan
|
||||
if self.kind is StageKind.ATTENTION:
|
||||
publish_attention(plan._publish_lora_layout)
|
||||
if plan._sp_steps is not None and plan.is_first_layer:
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
else:
|
||||
publish_ffn(plan._publish_lora_layout, self.entry(forward_batch).input_rows)
|
||||
return self._prepare(hidden_states, residual, forward_batch, **call)
|
||||
|
||||
def _prepare_attention(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
quant_format: str = "",
|
||||
capture_output: Optional[Callable] = None,
|
||||
):
|
||||
# Aux consumers need a materialized output before the input norm.
|
||||
# Complete communication here, preserving the existing add+norm kernel
|
||||
# and its FP32 accumulation. Its residual result also supplies capture.
|
||||
capture_before_read = capture_output is not None and (
|
||||
(stream.residual is None and stream.pending is None)
|
||||
or post_residual_addition is not None
|
||||
)
|
||||
if capture_output is not None:
|
||||
hidden_states = stream.complete(hidden_states)
|
||||
if capture_before_read:
|
||||
# Embeddings precede enter; HF deepstack capture precedes the
|
||||
# extra addition. Neither is the residual returned by the read.
|
||||
value, previous = stream.finish(hidden_states)
|
||||
if previous is None:
|
||||
capture_output(value)
|
||||
else:
|
||||
capture_output(value + previous, owned=True)
|
||||
hidden_states, stream = self._prepare_input(
|
||||
hidden_states,
|
||||
stream,
|
||||
forward_batch,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
entry = self.entry(forward_batch)
|
||||
if capture_output is not None and not capture_before_read:
|
||||
value = stream.residual
|
||||
move = entry.capture_move
|
||||
if move is not None:
|
||||
value = move(value, forward_batch=forward_batch)
|
||||
keeps_input = entry.capture_preserves_residual
|
||||
capture_output(
|
||||
value,
|
||||
owned=entry.capture_move_allocates
|
||||
or (
|
||||
move is None
|
||||
and keeps_input is not None
|
||||
and keeps_input(value, forward_batch)
|
||||
),
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
residual_value = stream.residual
|
||||
move = entry.input_move
|
||||
gathered_last_layer_output = (
|
||||
residual_value
|
||||
if move is None
|
||||
else move(
|
||||
hidden_states=residual_value,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
)
|
||||
# Input gathers allocate fresh DP buffers.
|
||||
# Without a gather this is the mutable residual, so retention copies.
|
||||
captured_last_layer_outputs.capture(
|
||||
gathered_last_layer_output,
|
||||
owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
or (
|
||||
move is None
|
||||
and entry.capture_preserves_residual is not None
|
||||
and entry.capture_preserves_residual(residual_value, forward_batch)
|
||||
),
|
||||
)
|
||||
return hidden_states, stream
|
||||
@@ -559,7 +559,12 @@ def draft_model_build_scope():
|
||||
original_scope = moe.in_speculative_scope
|
||||
try:
|
||||
moe.in_speculative_scope = True
|
||||
yield
|
||||
# Boundaries capture this resolved preference while the draft builds.
|
||||
# Restoring it cannot change an already constructed target plan.
|
||||
with get_exec().comm.override(
|
||||
boundary_reduction=get_spec().speculative_boundary_reduction
|
||||
):
|
||||
yield
|
||||
finally:
|
||||
moe.in_speculative_scope = original_scope
|
||||
moe.disable_shared_experts_fusion = original_fusion
|
||||
@@ -759,10 +764,8 @@ def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool:
|
||||
|
||||
Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one.
|
||||
"""
|
||||
return (
|
||||
should_skip_mlp_all_reduce()
|
||||
or should_use_dp_reduce_scatterv()
|
||||
or post_experts_output_is_complete(is_tp_path=is_tp_path)
|
||||
return should_skip_mlp_all_reduce() or post_experts_output_is_complete(
|
||||
is_tp_path=is_tp_path
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -565,7 +565,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
dp_prefill_cuda_graph_max_prefix_len: int = 0
|
||||
global_forward_mode: Optional[ForwardMode] = None
|
||||
|
||||
# Owned only while this invocation executes its local decoder stack.
|
||||
# Current layer-stack invocation; each TBO child owns a separate stream.
|
||||
residual_stream: Optional[ResidualStream] = None
|
||||
|
||||
# For two-batch overlap
|
||||
|
||||
@@ -36,9 +36,10 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateList,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -673,15 +674,6 @@ class BailingMoEBlock(nn.Module):
|
||||
config, layer_id=layer_id + 1, is_nextn=False
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = BailingMoESparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -706,11 +698,21 @@ class BailingMoEBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(
|
||||
@@ -725,18 +727,14 @@ class BailingMoEBlock(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
captured_last_layer_outputs=captured_last_layer_outputs,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -746,17 +744,17 @@ class BailingMoEBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class BailingMoEModel(nn.Module):
|
||||
@@ -821,51 +819,36 @@ class BailingMoEModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
captured_last_layer_outputs=(
|
||||
aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None
|
||||
),
|
||||
captured_last_layer_outputs=aux_hidden_states
|
||||
if getattr(layer, "_is_layer_to_capture", False)
|
||||
else None,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -16,8 +16,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -814,26 +815,28 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=is_moe_layer,
|
||||
is_previous_layer_sparse=is_previous_moe_layer,
|
||||
is_next_layer_sparse=is_next_layer_moe_layer,
|
||||
)
|
||||
|
||||
qkv_latent_func = (
|
||||
self.attention.prepare_qkv_latent
|
||||
if config.attention_type == 1 and self.use_mla
|
||||
else None
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=False,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{"qkv_latent_func": qkv_latent_func},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_sparse=is_next_layer_moe_layer,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(
|
||||
@@ -848,13 +851,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 1 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
@@ -875,16 +875,15 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 2 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
# logger.warning(
|
||||
# f"===={self.layer_id=}, 3 shape= {hidden_states.shape}, {residual.shape}"
|
||||
# )
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
@staticmethod
|
||||
def shared_moe_coefficient_loader(
|
||||
@@ -981,12 +980,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = inputs_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
|
||||
@@ -999,30 +998,20 @@ class BailingMoELinearModel(nn.Module):
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@ class BailingMoEModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
if self.is_hybrid:
|
||||
device = input_ids.device
|
||||
zero_allocator = BumpAllocator(
|
||||
@@ -172,26 +172,20 @@ class BailingMoEModelNextN(nn.Module):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
hidden_states, residual = self.decoder(
|
||||
hidden_states = self.decoder(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = self.decoder(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is not None:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.final_layernorm
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -30,9 +30,10 @@ from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1148,27 +1149,27 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
self.input_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(self.hidden_size, eps=rms_norm_eps)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
# NextN wraps a single decoder layer whose checkpoint prefix is the
|
||||
# post-model layer id. Treat it as a one-layer model for scatter-mode
|
||||
# planning so A2A/DeepEP outputs are gathered before logits.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=is_moe_layer,
|
||||
is_previous_layer_sparse=is_previous_moe_layer,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=(
|
||||
self.attention.prepare_qkv_latent
|
||||
if self.attention_type == 1 and self.use_mla
|
||||
else None
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"qkv_latent_func": self.attention.prepare_qkv_latent
|
||||
if self.attention_type == 1 and self.use_mla
|
||||
else None
|
||||
},
|
||||
),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == (1 if is_nextn else config.num_hidden_layers) - 1,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -1177,18 +1178,12 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
zero_allocator: BumpAllocator,
|
||||
capture_output=None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
@@ -1213,11 +1208,10 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if not (
|
||||
enable_moe_dense_fully_dp()
|
||||
and (not self.is_layer_sparse)
|
||||
@@ -1227,9 +1221,9 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
@staticmethod
|
||||
def shared_moe_coefficient_loader(
|
||||
@@ -1347,12 +1341,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = hidden_states.device
|
||||
@@ -1386,41 +1380,30 @@ class BailingMoELinearModel(nn.Module):
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
residual=residual,
|
||||
zero_allocator=zero_allocator,
|
||||
capture_output=capture_output
|
||||
if capture_aux
|
||||
and i > self.start_layer
|
||||
and i - 1 in self.layers_to_capture
|
||||
and (i - 1 in self.layers_to_capture)
|
||||
else None,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
capture_output
|
||||
if capture_aux and self.end_layer - 1 in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
capture_output=capture_output
|
||||
if capture_aux and self.end_layer - 1 in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
if capture_aux and len(dspark_aux_hidden_states) > 0:
|
||||
return hidden_states, dspark_aux_hidden_states
|
||||
|
||||
@@ -3794,7 +3794,12 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and get_parallel().tp_size == get_parallel().attn_dp_size
|
||||
)
|
||||
mlp_reduce_scatter = _use_cp or _use_reduce_scatterv or _use_reduce_scatter
|
||||
mlp_reduce_scatter = (
|
||||
_use_cp
|
||||
or _use_reduce_scatterv
|
||||
or _use_reduce_scatter
|
||||
or (_use_tp_moe_gather and should_use_dp_reduce_scatterv())
|
||||
)
|
||||
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): compute the replicated shared expert
|
||||
# on LOCAL hidden before the gather and add it back after the combine
|
||||
# (reduce_scatterv OR dp_scatter), instead of on the gathered global buffer.
|
||||
|
||||
@@ -11,7 +11,12 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.output import OutputTransform
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -90,9 +95,11 @@ class FalconH1MLP(nn.Module):
|
||||
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
x = x * self.down_multiplier
|
||||
return x
|
||||
|
||||
def scale_output(self, x):
|
||||
return x * self.down_multiplier
|
||||
|
||||
|
||||
class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
@@ -199,14 +206,6 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
self.feed_forward = FalconH1MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
@@ -223,12 +222,24 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.pre_ff_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
output=OutputTransform(
|
||||
self.feed_forward.scale_output, before_reduce_scatter=True
|
||||
),
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.pre_ff_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
@@ -320,13 +331,10 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
**kwargs: Any,
|
||||
):
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
# Attention block
|
||||
@@ -355,12 +363,11 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_states = attention_hidden_states + mamba_hidden_states
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
@@ -422,26 +429,22 @@ class FalconH1Model(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
layer_id=i,
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.final_layernorm
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -31,8 +31,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -570,14 +571,6 @@ class GptOssDecoderLayer(nn.Module):
|
||||
is_previous_layer_sparse = True
|
||||
is_next_layer_sparse = True
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = GptOssSparseMoeBlock(
|
||||
layer_id=self.layer_id,
|
||||
@@ -606,11 +599,21 @@ class GptOssDecoderLayer(nn.Module):
|
||||
x_pad_to_multiple=post_attn_pad_multiple,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -618,16 +621,10 @@ class GptOssDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
capture_output=None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -637,15 +634,14 @@ class GptOssDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class GptOssModel(nn.Module):
|
||||
@@ -708,12 +704,12 @@ class GptOssModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
# Capture hidden-state boundaries: boundary 0 is the embedding output,
|
||||
# and boundary i + 1 is the output after transformer block i.
|
||||
@@ -721,44 +717,30 @@ class GptOssModel(nn.Module):
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = last_layer.layer_communicator.norm_output(
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
capture_output=(
|
||||
aux_hidden_states.append
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None
|
||||
),
|
||||
capture_output=aux_hidden_states.capture
|
||||
if self.end_layer in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
elif self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
residual_batch.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -32,9 +32,10 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
enable_moe_dense_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -613,14 +614,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
|
||||
is_next_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id + 1)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if self.is_layer_sparse:
|
||||
self.mlp = LLaDA2MoeSparseMoeBlock(
|
||||
layer_id=layer_id,
|
||||
@@ -645,12 +638,21 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
|
||||
self.post_attention_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
|
||||
@@ -663,12 +665,9 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states=hidden_states, forward_batch=forward_batch
|
||||
)
|
||||
|
||||
hidden_states = self.attention(
|
||||
@@ -677,17 +676,17 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class LLaDA2MoeModel(nn.Module):
|
||||
@@ -747,44 +746,26 @@ class LLaDA2MoeModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -25,7 +25,12 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Llama4TextConfig
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -421,20 +426,19 @@ class Llama4DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=is_moe_layer,
|
||||
is_previous_layer_sparse=is_previous_moe_layer,
|
||||
is_next_layer_sparse=is_next_moe_layer,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=is_moe_layer,
|
||||
next_sparse=is_next_moe_layer,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=is_previous_moe_layer, next_sparse=is_moe_layer)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _is_moe_layer(self, layer_id: int) -> bool:
|
||||
@@ -453,10 +457,10 @@ class Llama4DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states, forward_batch, capture_output=capture_output
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
@@ -466,16 +470,15 @@ class Llama4DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
# Fully Connected
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Llama4Model(nn.Module):
|
||||
@@ -519,28 +522,22 @@ class Llama4Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
residual_batch.start(forward_batch)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(len(self.layers)):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
self.layers[i].layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
@@ -568,29 +565,21 @@ class Llama4ForCausalLM(LlamaForCausalLM):
|
||||
):
|
||||
start, end = split_interval
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
forward_batch.hidden_states = (
|
||||
self.model.embed_tokens(input_ids)
|
||||
if input_embeds is None
|
||||
else input_embeds
|
||||
)
|
||||
for layer in self.model.layers[start:end]:
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
forward_batch.hidden_states = layer(
|
||||
positions, forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
if end != self.model.config.num_hidden_layers:
|
||||
return None
|
||||
hidden, residual = LayerCommunicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.residual = None
|
||||
if residual is None:
|
||||
forward_batch.hidden_states = self.model.norm(hidden)
|
||||
else:
|
||||
forward_batch.hidden_states, _ = self.model.norm(hidden, residual)
|
||||
return self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
replace_prefix,
|
||||
replace_substrings,
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import make_layer_communicator
|
||||
from sglang.srt.models.nemotron_h_utils import make_stage_boundary
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
@@ -375,8 +375,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
mixer's attention partial sum, or a value that is complete or carries the
|
||||
sum an FFN before it left."""
|
||||
|
||||
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.stage = make_stage_boundary(
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
@@ -384,15 +384,12 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
self,
|
||||
*,
|
||||
hidden_states,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.stage.prepare(hidden_states, forward_batch)
|
||||
with self.stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -426,7 +423,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
self._init_stage_boundary(config, layer_idx)
|
||||
|
||||
|
||||
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -449,7 +446,7 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
self._init_stage_boundary(config, layer_idx)
|
||||
|
||||
|
||||
class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
@@ -457,8 +454,8 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
output all-reduce to that stage's input; otherwise it reduces the output
|
||||
itself unless the fused kernel takes the sum into the next input norm."""
|
||||
|
||||
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
def _init_stage_boundary(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.stage = make_stage_boundary(
|
||||
self.norm, pattern=config.hybrid_override_pattern, layer_idx=layer_idx
|
||||
)
|
||||
|
||||
@@ -466,20 +463,17 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
self,
|
||||
*,
|
||||
hidden_states,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.stage.prepare(hidden_states, forward_batch)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
with self.layer_communicator.mixer_exit(forward_batch) as mixer_exit:
|
||||
with self.stage.exit(forward_batch) as mixer_exit:
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, mixer_exit.skips_reduction
|
||||
)
|
||||
return mixer_exit.finish(hidden_states, residual)
|
||||
return mixer_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
@@ -507,7 +501,7 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
self._init_stage_boundary(config, layer_idx)
|
||||
|
||||
def _forward_mamba(
|
||||
self,
|
||||
@@ -645,7 +639,7 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
self._init_stage_boundary(config, layer_idx)
|
||||
|
||||
def _forward_mixer(
|
||||
self,
|
||||
@@ -733,50 +727,38 @@ class NemotronHModel(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states, snapshot = self.layers[
|
||||
i
|
||||
].layer_communicator.capture_output(
|
||||
hidden_states, residual, at_input=True
|
||||
hidden_states, snapshot = self.layers[i].stage.capture_output(
|
||||
hidden_states, forward_batch, at_input=True
|
||||
)
|
||||
aux_hidden_states.append(snapshot)
|
||||
layer = self.layers[i]
|
||||
if not isinstance(layer, Layers):
|
||||
raise ValueError(f"Unknown layer type: {type(layer)}")
|
||||
hidden_states, residual = layer.forward(
|
||||
hidden_states = layer.forward(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
return residual_batch.to_pp(
|
||||
hidden_states, forward_batch, preserve_declared=True
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if self.end_layer in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
last_layer.layer_communicator.snapshot(hidden_states, residual)
|
||||
residual_batch.snapshot(hidden_states, forward_batch)
|
||||
)
|
||||
if residual is None:
|
||||
hidden_states = self.norm_f(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm_f(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm_f)
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -89,9 +89,8 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
|
||||
*,
|
||||
inputs_embeds: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
if self.has_start_projections:
|
||||
inputs_embeds_normed = self.enorm(inputs_embeds)
|
||||
previous_hidden_states_normed = self.hnorm(hidden_states)
|
||||
@@ -105,23 +104,22 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
|
||||
hidden_states, dim=-1
|
||||
)
|
||||
|
||||
hidden_states, residual = super().forward(
|
||||
if self.has_start_projections:
|
||||
hidden_states = residual_batch.written(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = super().forward(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
if self.has_end_norm:
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = self.layer_communicator.fold(
|
||||
hidden_states, residual
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = residual_batch.written(
|
||||
self.final_layernorm(hidden_states), forward_batch
|
||||
)
|
||||
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
@@ -173,9 +171,8 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
*,
|
||||
inputs_embeds: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
if self.has_start_projections:
|
||||
inputs_embeds_normed = self.enorm(inputs_embeds)
|
||||
previous_hidden_states_normed = self.hnorm(hidden_states)
|
||||
@@ -189,23 +186,22 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
hidden_states, dim=-1
|
||||
)
|
||||
|
||||
hidden_states, residual = super().forward(
|
||||
if self.has_start_projections:
|
||||
hidden_states = residual_batch.written(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = super().forward(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
if self.has_end_norm:
|
||||
hidden_states, residual = self.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = self.layer_communicator.fold(
|
||||
hidden_states, residual
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = residual_batch.written(
|
||||
self.final_layernorm(hidden_states), forward_batch
|
||||
)
|
||||
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class NemotronHMultiTokenPredictor(nn.Module):
|
||||
@@ -302,18 +298,15 @@ class NemotronHMultiTokenPredictor(nn.Module):
|
||||
inputs_embeds = self.get_input_embeddings(input_ids)
|
||||
|
||||
hidden_states = forward_batch.spec_info.hidden_states
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
|
||||
for i in range(self.pattern_len):
|
||||
if residual is None:
|
||||
residual = residual_batch.start(forward_batch)
|
||||
hidden_states, residual = self.layers[str(i)](
|
||||
hidden_states = self.layers[str(i)](
|
||||
inputs_embeds=inputs_embeds,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
return hidden_states
|
||||
return residual_batch.take_output(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class NemotronHForCausalLMMTP(NemotronHForCausalLM):
|
||||
|
||||
@@ -101,14 +101,12 @@ def layer_stage(pattern: str, layer_idx: int) -> LayerStage:
|
||||
)
|
||||
|
||||
|
||||
def make_layer_communicator(
|
||||
layer_norm: RMSNorm, *, pattern: str, layer_idx: int
|
||||
) -> LayerCommunicator:
|
||||
def make_stage_boundary(layer_norm: RMSNorm, *, pattern: str, layer_idx: int):
|
||||
"""The communicator of a layer that is one stage: only its own norm, and
|
||||
boundaries built from the stages next to it in the pattern."""
|
||||
stage = layer_stage(pattern, layer_idx)
|
||||
for_attn = stage.kind is StageKind.ATTENTION
|
||||
return LayerCommunicator(
|
||||
boundaries = LayerCommunicator(
|
||||
layer_facts=LayerFacts(
|
||||
is_layer_sparse=pattern[layer_idx] == MOE,
|
||||
is_last_layer=layer_idx == len(pattern) - 1,
|
||||
@@ -121,3 +119,5 @@ def make_layer_communicator(
|
||||
allow_reduce_scatter=not for_attn,
|
||||
stage=stage,
|
||||
)
|
||||
|
||||
return boundaries.attn if for_attn else boundaries.ffn
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
# Adapted from qwen2.py
|
||||
import logging
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
@@ -372,18 +376,14 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps, **norm_kwargs
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
is_previous_layer_sparse=False,
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -391,15 +391,15 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states = self.attn_stage.prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
post_residual_addition=post_residual_addition,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -409,9 +409,9 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
cache=(
|
||||
[self.mlp.gate_up_proj.weight, self.mlp.down_proj.weight]
|
||||
@@ -424,12 +424,12 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
else None
|
||||
),
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
if _is_npu and get_cmo_stream():
|
||||
wait_cmo_stream()
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen3Model(Qwen2Model):
|
||||
@@ -450,60 +450,39 @@ class Qwen3Model(Qwen2Model):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: torch.Tensor = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, PPProxyTensors]:
|
||||
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_embeds=None,
|
||||
pp_proxy_tensors=None,
|
||||
):
|
||||
if self.pp_group.is_first_rank:
|
||||
if input_embeds is None:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
residual_access.snapshot(hidden_states, residual)
|
||||
)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = self.layers[i](
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
hidden_states, residual = residual_access.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
return hidden_states, aux_hidden_states
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return (
|
||||
(hidden_states, aux_hidden_states) if aux_hidden_states else hidden_states
|
||||
)
|
||||
|
||||
|
||||
class Qwen3ForCausalLM(nn.Module):
|
||||
@@ -613,7 +592,7 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -621,25 +600,16 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
# decoder layer
|
||||
for i in range(start, end):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
forward_batch.hidden_states = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1205,24 +1206,23 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for layer_idx, layer in enumerate(
|
||||
self.layers[self.start_layer : self.end_layer]
|
||||
):
|
||||
layer_idx = layer_idx + self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
layer.layer_communicator.snapshot(
|
||||
hidden_states, residual, at_input=True
|
||||
)
|
||||
)
|
||||
capture_output = (
|
||||
aux_hidden_states.capture
|
||||
if layer_idx in self.layers_to_capture
|
||||
else None
|
||||
)
|
||||
|
||||
if self.use_hf_deepstack_order:
|
||||
# HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the
|
||||
@@ -1231,40 +1231,33 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
deepstack_embeds = self.get_deepstack_embeds(
|
||||
layer_idx - 1, input_deepstack_embeds
|
||||
)
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
post_residual_addition=deepstack_embeds,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
else:
|
||||
# Inference path: add deepstack directly to hidden_states at the end of the layer
|
||||
# (original, grounding-correct order).
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=capture_output,
|
||||
)
|
||||
if (
|
||||
input_deepstack_embeds is not None
|
||||
and layer_idx in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
hidden_states = residual_batch.add_to_output(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size],
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
|
||||
# Handle deepstack for the last processed layer (HF-order path only).
|
||||
last_deepstack = (
|
||||
self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds)
|
||||
@@ -1273,20 +1266,15 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self.norm,
|
||||
post_residual_addition=last_deepstack,
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(
|
||||
hidden_states, residual, post_residual_addition=last_deepstack
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -72,7 +72,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
make_ple_file_prefetcher,
|
||||
make_ple_file_rss_trimmer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, logger
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
@@ -1439,14 +1439,16 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
attn_tp_chunks = list(hidden_states.tensor_split(attn_tp_size))
|
||||
hidden_states = attn_tp_chunks[get_parallel().attn_tp_rank].contiguous()
|
||||
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
use_reduce_scatterv = use_dp_moe_gather and should_use_dp_reduce_scatterv()
|
||||
with get_forward().scoped(mlp_reduce_scatter=use_reduce_scatterv):
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if use_dp_moe_gather:
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
if should_use_dp_reduce_scatterv():
|
||||
if use_reduce_scatterv:
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
|
||||
@@ -1456,36 +1456,26 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
|
||||
start, end = split_interval
|
||||
|
||||
if start == 0:
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.word_embeddings(input_ids)
|
||||
else:
|
||||
forward_batch.hidden_states = input_embeds
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
forward_batch.hidden_states = layer(
|
||||
positions, forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
forward_batch.hidden_states, forward_batch.residual = self.model.layers[
|
||||
end - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
forward_batch.hidden_states = residual_batch.finish(
|
||||
forward_batch.hidden_states, forward_batch
|
||||
)
|
||||
forward_batch.hidden_states = residual_batch.norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
if forward_batch.residual is None:
|
||||
hidden_states = self.model.norm(forward_batch.hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
|
||||
return self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
|
||||
@@ -11,7 +11,11 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -291,19 +295,17 @@ class SDARBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=False,
|
||||
is_previous_layer_sparse=False,
|
||||
is_next_layer_sparse=False,
|
||||
)
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=False,
|
||||
next_sparse=False,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -311,13 +313,8 @@ class SDARBlock(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -326,17 +323,14 @@ class SDARBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class SDARModel(nn.Module):
|
||||
@@ -404,37 +398,25 @@ class SDARModel(nn.Module):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch, allow_missing_residual=True
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layers[
|
||||
self.end_layer - 1
|
||||
].layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states, residual = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -14,8 +14,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -356,19 +357,16 @@ class SDARMoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=True,
|
||||
is_previous_layer_sparse=True,
|
||||
is_next_layer_sparse=True,
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=True, next_sparse=True),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(sparse=True, next_sparse=True)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -376,12 +374,9 @@ class SDARMoeBlock(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -390,18 +385,17 @@ class SDARMoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class SDARMoeModel(nn.Module):
|
||||
@@ -470,34 +464,24 @@ class SDARMoeModel(nn.Module):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
hidden_states = self.layers[self.start_layer].attn_stage.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
last_layer = self.layers[self.end_layer - 1]
|
||||
hidden_states, residual = last_layer.layer_communicator.finish_layer_stack(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
preserve_declared=not self.pp_group.is_last_rank,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
hidden_states, residual = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -16,13 +16,14 @@ from sglang.srt.configs.step3_vl import (
|
||||
Step3VisionEncoderConfig,
|
||||
Step3VLConfig,
|
||||
)
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -40,6 +41,7 @@ from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import reduce_moe_output
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -78,6 +80,7 @@ class Step3TextMLP(nn.Module):
|
||||
hidden_act: str,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -93,6 +96,7 @@ class Step3TextMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -163,8 +167,6 @@ class Step3TextMoEMLP(nn.Module):
|
||||
hidden_states=hidden_states, topk_output=topk_output
|
||||
)
|
||||
|
||||
if self.tp_size > 1:
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -339,14 +341,6 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
)
|
||||
self.is_next_layer_sparse = True if layer_id + 1 in moe_layers_idx else False
|
||||
|
||||
self.layer_facts = LayerFacts.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=self.is_previous_layer_sparse,
|
||||
is_next_layer_sparse=self.is_next_layer_sparse,
|
||||
)
|
||||
|
||||
if not self.is_layer_sparse:
|
||||
self.mlp = Step3TextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -370,6 +364,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
hidden_act="silu",
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("share_expert", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.moe = Step3TextMoEMLP(
|
||||
@@ -379,11 +374,21 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.layer_communicator = LayerCommunicator(
|
||||
layer_facts=self.layer_facts,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
self.attn_stage, self.ffn_stage = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
sparse=self.is_layer_sparse,
|
||||
next_sparse=self.is_next_layer_sparse,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn(
|
||||
sparse=self.is_previous_layer_sparse, next_sparse=self.is_layer_sparse
|
||||
)
|
||||
if layer_id != 0
|
||||
else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def moe_mlp_forward(self, hidden_states):
|
||||
@@ -393,19 +398,16 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
hidden_states += self.share_expert(h)
|
||||
else:
|
||||
hidden_states = self.moe(hidden_states)
|
||||
return hidden_states
|
||||
return reduce_moe_output(hidden_states)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.attn_stage.prepare(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states = self.self_attn(
|
||||
@@ -414,17 +416,16 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.attn_stage.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_stage.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_stage.exit(forward_batch) as ffn_exit:
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Step3TextModel(nn.Module):
|
||||
@@ -472,22 +473,15 @@ class Step3TextModel(nn.Module):
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
|
||||
residual = residual_batch.start(forward_batch)
|
||||
residual_batch.start(forward_batch)
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = residual_batch.finish(hidden_states, forward_batch)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = residual_batch.norm(hidden_states, forward_batch, self.norm)
|
||||
return hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Small fixtures around production boundaries; no parallel layout recipes."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.communicator.construction import BatchVariant, StagePlan
|
||||
from sglang.srt.layers.communicator.factories import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import PLAIN_RESIDUAL
|
||||
from sglang.srt.layers.communicator.stage import StageCommunicator
|
||||
|
||||
|
||||
def make_test_stages(
|
||||
*,
|
||||
attention_norm,
|
||||
ffn_norm,
|
||||
first=False,
|
||||
last=False,
|
||||
sparse=False,
|
||||
previous_sparse=False,
|
||||
next_sparse=False,
|
||||
residual=PLAIN_RESIDUAL,
|
||||
output=None,
|
||||
**options,
|
||||
):
|
||||
previous = (
|
||||
None
|
||||
if first
|
||||
else declare_ffn(
|
||||
sparse=previous_sparse, next_sparse=sparse, update=residual.ffn_update
|
||||
)
|
||||
)
|
||||
attention = declare_attn(
|
||||
read=residual.attention_read, update=residual.attention_update
|
||||
)
|
||||
ffn = declare_ffn(
|
||||
sparse=sparse,
|
||||
next_sparse=next_sparse,
|
||||
read=residual.ffn_read,
|
||||
update=residual.ffn_update,
|
||||
output=output,
|
||||
)
|
||||
common = {
|
||||
key: options.pop(key)
|
||||
for key in ("force_layernorm_before_dp_gather", "fusions")
|
||||
if key in options
|
||||
}
|
||||
attn, ffn = make_stages(
|
||||
(
|
||||
attention,
|
||||
attention_norm,
|
||||
dict(
|
||||
residual_in_hidden=residual.ffn_update.at_producer, **common, **options
|
||||
),
|
||||
),
|
||||
(ffn, ffn_norm, common),
|
||||
previous=previous,
|
||||
terminal=last,
|
||||
)
|
||||
return SimpleNamespace(attn=attn, ffn=ffn)
|
||||
|
||||
|
||||
def stub_plan():
|
||||
"""Supply only defaults for tests that inject preselected paths/kernels."""
|
||||
plan = StagePlan.__new__(StagePlan)
|
||||
plan.norm = None
|
||||
plan.fusions = None
|
||||
plan._publish_lora_layout = False
|
||||
plan._fusion_rows = None
|
||||
plan._cp_steps = plan._sp_steps = plan._input_scattered_steps = None
|
||||
plan.is_first_layer = False
|
||||
plan.residual_in_hidden = False
|
||||
# Keep production variant selection while tests replace individual paths.
|
||||
plan._batch_steps = lambda batch: {
|
||||
BatchVariant.ORDINARY: plan._steps,
|
||||
BatchVariant.CONTEXT_PARALLEL: plan._cp_steps,
|
||||
BatchVariant.INPUT_SCATTERED: plan._input_scattered_steps,
|
||||
BatchVariant.SEQUENCE_PARALLEL: plan._sp_steps,
|
||||
}[plan._variant(batch)]
|
||||
return plan
|
||||
|
||||
|
||||
def stub_stage(plan, kind):
|
||||
"""Keep the same concrete boundary when a test patches its methods."""
|
||||
stages = plan.__dict__.setdefault("_stub_stages", {})
|
||||
if kind not in stages:
|
||||
stages[kind] = StageCommunicator(plan, kind, plan.norm)
|
||||
return stages[kind]
|
||||
|
||||
|
||||
def sp_region_steps():
|
||||
"""Local SP rows with no owed sum, for tests of activation and exits."""
|
||||
from sglang.srt.layers.communicator import (
|
||||
NORM_QUANT_READ,
|
||||
NORM_READ,
|
||||
BoundarySteps,
|
||||
CommunicateSummableTensorPairFn,
|
||||
EdgeDecl,
|
||||
Layout,
|
||||
StageEntry,
|
||||
StageInput,
|
||||
StageOutput,
|
||||
TokenAxis,
|
||||
make_boundary,
|
||||
)
|
||||
from sglang.srt.layers.communicator.ops import _hand_qkv_hook_its_input
|
||||
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
output = StageOutput(rows)
|
||||
|
||||
def entry(read, handoff=None):
|
||||
selected = make_boundary(
|
||||
EdgeDecl(output, StageInput(rows, read=read), rows, rows)
|
||||
)
|
||||
return StageEntry(selected.prepare, rows, handoff=handoff)
|
||||
|
||||
return BoundarySteps(
|
||||
entry(NORM_QUANT_READ, _hand_qkv_hook_its_input),
|
||||
entry(NORM_READ),
|
||||
output,
|
||||
CommunicateSummableTensorPairFn._trivial,
|
||||
False,
|
||||
)
|
||||
@@ -8,14 +8,40 @@ the name, not only into the package.
|
||||
import contextlib
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.layers import communicator
|
||||
from sglang.srt.layers.communicator import boundary, layer, layout, ops, output
|
||||
from sglang.srt.layers import communicator as communicator
|
||||
from sglang.srt.layers.communicator import (
|
||||
boundary,
|
||||
construction,
|
||||
exit,
|
||||
factories,
|
||||
layer,
|
||||
layout,
|
||||
legacy_stage,
|
||||
ops,
|
||||
output,
|
||||
)
|
||||
from sglang.srt.layers.communicator import residual as residual_contract
|
||||
from sglang.srt.layers.communicator.adapters import attention
|
||||
from sglang.srt.layers.communicator.residual import access, add_norm, mhc
|
||||
from sglang.srt.layers.communicator import (
|
||||
stage,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters import attention, branch, lora
|
||||
from sglang.srt.layers.communicator.fusions import allreduce
|
||||
from sglang.srt.layers.communicator.residual import (
|
||||
access,
|
||||
add_norm,
|
||||
batch,
|
||||
mhc,
|
||||
stream,
|
||||
)
|
||||
|
||||
COMMUNICATOR_MODULES = (
|
||||
layer,
|
||||
legacy_stage,
|
||||
communicator,
|
||||
allreduce,
|
||||
exit,
|
||||
branch,
|
||||
lora,
|
||||
layout,
|
||||
output,
|
||||
attention,
|
||||
@@ -25,7 +51,11 @@ COMMUNICATOR_MODULES = (
|
||||
mhc,
|
||||
ops,
|
||||
boundary,
|
||||
layer,
|
||||
construction,
|
||||
factories,
|
||||
stage,
|
||||
batch,
|
||||
stream,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user