1736 lines
66 KiB
Python
1736 lines
66 KiB
Python
# Copyright 2023-2024 SGLang Team
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
import logging
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from enum import Enum, auto
|
|
from functools import cached_property, partial
|
|
from typing import Callable, Dict, Optional, Tuple, Union
|
|
|
|
import torch
|
|
|
|
from sglang.srt.distributed import (
|
|
attention_tensor_model_parallel_all_reduce,
|
|
attention_tensor_model_parallel_quant_all_reduce,
|
|
tensor_model_parallel_all_reduce,
|
|
)
|
|
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
|
use_symmetric_memory,
|
|
)
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.layers import layernorm_sp
|
|
from sglang.srt.layers.attention.dsa.utils import (
|
|
dsa_use_prefill_cp,
|
|
is_dsa_enable_prefill_cp,
|
|
)
|
|
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
|
from sglang.srt.layers.boundary_layout import Layout, TokenAxis
|
|
from sglang.srt.layers.cp.utils import (
|
|
is_mla_cp_active,
|
|
is_mla_cp_enabled,
|
|
)
|
|
from sglang.srt.layers.dp_attention import (
|
|
attn_tp_all_gather_into_tensor,
|
|
attn_tp_reduce_scatter_tensor,
|
|
can_use_dp_reduce_scatter,
|
|
dp_gather_partial,
|
|
dp_gather_replicate,
|
|
dp_reduce_scatter_tensor,
|
|
dp_scatter,
|
|
get_dp_global_num_tokens,
|
|
get_global_dp_buffer,
|
|
get_local_dp_buffer,
|
|
get_moe_cp_rank,
|
|
get_moe_cp_size,
|
|
is_allocation_symmetric,
|
|
is_dp_attention_enabled,
|
|
is_enable_moe_cp_allgather,
|
|
moe_cp_all_gather_into_tensor,
|
|
)
|
|
from sglang.srt.layers.flashinfer_comm_fusion import (
|
|
is_flashinfer_allreduce_unavailable,
|
|
uses_cutedsl_ar_fusion,
|
|
)
|
|
from sglang.srt.layers.moe import (
|
|
can_merge_post_experts_all_reduce,
|
|
deferred_post_experts_all_reduce,
|
|
get_moe_a2a_backend,
|
|
should_use_dp_reduce_scatterv,
|
|
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
|
)
|
|
from sglang.srt.layers.quantization.fp8_utils import (
|
|
_use_aiter_bpreshuffle_gfx95,
|
|
materialize_bpreshuffle_fp8_scale_tuple,
|
|
)
|
|
from sglang.srt.model_executor.cuda_graph_config import (
|
|
Backend,
|
|
Phase,
|
|
check_cuda_graph_backend,
|
|
)
|
|
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
|
from sglang.srt.runtime_context import (
|
|
get_exec,
|
|
get_forward,
|
|
get_parallel,
|
|
get_platform,
|
|
get_spec,
|
|
)
|
|
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
|
from sglang.srt.utils import (
|
|
get_bool_env_var,
|
|
is_cuda,
|
|
is_flashinfer_available,
|
|
is_gfx95_supported,
|
|
is_gfx1250_supported,
|
|
is_hip,
|
|
is_npu,
|
|
)
|
|
|
|
_is_cuda = is_cuda()
|
|
_is_flashinfer_available = is_flashinfer_available()
|
|
_is_sm90_supported = _is_cuda and get_platform().is_sm90
|
|
_is_sm100_supported = _is_cuda and get_platform().is_sm100
|
|
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
|
_is_gfx95_supported = is_gfx95_supported()
|
|
_is_gfx1250_supported = is_gfx1250_supported()
|
|
_is_npu = is_npu()
|
|
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
|
|
|
if _use_aiter:
|
|
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype
|
|
|
|
if _is_gfx1250_supported:
|
|
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
|
else:
|
|
from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant
|
|
from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant
|
|
|
|
if _is_gfx95_supported:
|
|
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
|
|
|
from sglang.srt.layers.quantization.rocm_mxfp4_utils import (
|
|
fused_rms_mxfp4_quant,
|
|
)
|
|
elif _is_npu:
|
|
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
|
|
|
|
|
def _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
epsilon: float,
|
|
residual: Optional[torch.Tensor] = None,
|
|
):
|
|
"""Fused (optional residual-add +) RMSNorm + FP8 per-token quantization.
|
|
|
|
Only used with the aiter (ROCm) backend.
|
|
|
|
Args:
|
|
residual: if provided, computes hidden_states + residual before RMSNorm
|
|
and returns updated residual_out as second element.
|
|
|
|
Returns:
|
|
If residual is None: (out_fp8, scale)
|
|
If residual provided: ((out_fp8, scale), residual_out)
|
|
"""
|
|
if _is_gfx1250_supported:
|
|
# per-token quant == group quant with group_size == hidden size, giving
|
|
# an (M, 1) scale.
|
|
N = hidden_states.shape[-1]
|
|
(out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant(
|
|
hidden_states,
|
|
weight,
|
|
epsilon,
|
|
group_size=N,
|
|
dtype_quant=_aiter_fp8_dtype,
|
|
res1=residual,
|
|
)
|
|
if residual is not None:
|
|
return (out_fp8, scale), residual_out
|
|
return (out_fp8, scale)
|
|
|
|
M, N = hidden_states.shape
|
|
out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device)
|
|
scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device)
|
|
if residual is not None:
|
|
residual_out = torch.empty_like(hidden_states)
|
|
_aiter_add_rmsnorm_quant(
|
|
out_fp8,
|
|
hidden_states,
|
|
residual,
|
|
residual_out,
|
|
scale,
|
|
weight,
|
|
epsilon,
|
|
0, # group_size=0 → per-token
|
|
)
|
|
return (out_fp8, scale.unsqueeze(1)), residual_out
|
|
else:
|
|
_aiter_rmsnorm_quant(
|
|
out_fp8,
|
|
hidden_states,
|
|
scale,
|
|
weight,
|
|
epsilon,
|
|
0, # group_size=0 → per-token
|
|
)
|
|
return (out_fp8, scale.unsqueeze(1))
|
|
|
|
|
|
# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465
|
|
# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True).
|
|
FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048
|
|
|
|
|
|
def apply_flashinfer_allreduce_fusion(batch_size: int):
|
|
return (
|
|
# NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion
|
|
# Ref: https://github.com/sgl-project/sglang/issues/17237
|
|
(_is_sm90_supported or _is_sm100_supported)
|
|
and _is_flashinfer_available
|
|
and not is_dp_attention_enabled()
|
|
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
|
# cutedsl runs its own fused path from the fusion communicator.
|
|
and not uses_cutedsl_ar_fusion()
|
|
and not is_flashinfer_allreduce_unavailable()
|
|
# Symbolic size checks stay last: under Dynamo tracing they guard on
|
|
# the dynamic token dim, so statically-off configs must short-circuit
|
|
# before reaching them.
|
|
and batch_size > 0
|
|
and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
|
)
|
|
|
|
|
|
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
|
|
comm = get_exec().comm
|
|
if not comm.enable_aiter_allreduce_fusion:
|
|
return False
|
|
if forward_mode.is_extend_or_draft_extend_or_mixed():
|
|
return not comm.disable_aiter_allreduce_fusion_in_prefill
|
|
return not comm.disable_aiter_allreduce_fusion_in_decode
|
|
|
|
|
|
def apply_aiter_all_reduce_fusion(
|
|
input_tensor: torch.Tensor, forward_batch: ForwardBatch
|
|
):
|
|
n = input_tensor.shape[-1]
|
|
total_bytes = input_tensor.numel() * input_tensor.element_size()
|
|
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
|
|
return (
|
|
_use_aiter
|
|
and total_bytes > 0
|
|
and n <= 16384
|
|
and total_bytes <= 8 * 1024 * 8192
|
|
and get_parallel().tp_size != 6
|
|
and not is_dp_attention_enabled()
|
|
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
|
)
|
|
|
|
|
|
class ScatterMode(Enum):
|
|
"""
|
|
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
|
|
Model input/output: [ab, ab, cd, cd] for four ranks respectively
|
|
SCATTERED: [a, b, c, d]
|
|
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
|
|
FULL: [abcd, abcd, abcd, abcd]
|
|
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
|
|
"""
|
|
|
|
SCATTERED = auto()
|
|
TP_ATTN_FULL = auto()
|
|
FULL = auto()
|
|
MOE_FULL = auto()
|
|
|
|
@staticmethod
|
|
def model_input_output():
|
|
"""The scatter mode for model forward pass input and output data"""
|
|
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
|
|
return ScatterMode.SCATTERED
|
|
|
|
return ScatterMode.TP_ATTN_FULL
|
|
|
|
|
|
class AttentionInputs:
|
|
def __init__(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
qkv_latent_func: Callable,
|
|
*,
|
|
is_pre_gathered: bool = False,
|
|
):
|
|
self.hidden_states_local = hidden_states
|
|
self.forward_batch = forward_batch
|
|
self.qkv_latent_func = qkv_latent_func
|
|
self.hidden_states_ = None
|
|
self.qkv_latent_ = None
|
|
# When True, hidden_states_local is already attn_tp-gathered upstream
|
|
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
|
|
self.is_pre_gathered = is_pre_gathered
|
|
|
|
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
|
|
total_tokens = forward_batch.input_ids.shape[0]
|
|
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
|
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
|
return output
|
|
|
|
def fetch_qkv_latent(self):
|
|
if self.qkv_latent_ is not None:
|
|
return self.qkv_latent_
|
|
assert self.qkv_latent_func is not None
|
|
self.qkv_latent_ = self.qkv_latent_func(
|
|
self.hidden_states_local, self.forward_batch
|
|
)
|
|
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
|
self.qkv_latent_ = self.tp_all_gather_hidden_states(
|
|
self.qkv_latent_, self.forward_batch
|
|
)
|
|
return self.qkv_latent_
|
|
|
|
def fetch_hidden_states(self):
|
|
if self.hidden_states_ is not None:
|
|
return self.hidden_states_
|
|
self.hidden_states_ = self.hidden_states_local
|
|
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
|
self.hidden_states_ = self.tp_all_gather_hidden_states(
|
|
self.hidden_states_, self.forward_batch
|
|
)
|
|
return self.hidden_states_
|
|
|
|
|
|
class AttnTpContext:
|
|
def __init__(self):
|
|
self.allow_input_scattered = False
|
|
self.is_dsa = False
|
|
|
|
def init_context(self, q_lora_rank, is_dsa, is_mhc=False):
|
|
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
|
|
# cannot use scattered inputs.
|
|
self.is_dsa = is_dsa
|
|
self.allow_input_scattered = (
|
|
get_parallel().enable_attn_tp_input_scattered
|
|
and (_is_cuda or _is_npu)
|
|
and q_lora_rank is not None
|
|
and (is_mhc or not is_dsa)
|
|
and get_parallel().tp_size > 1
|
|
and not is_dp_attention_enabled()
|
|
and get_moe_a2a_backend().is_none()
|
|
and not enable_moe_dense_fully_dp()
|
|
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
|
and get_spec().speculative_algorithm != "EAGLE3"
|
|
)
|
|
if get_parallel().enable_attn_tp_input_scattered:
|
|
if not self.allow_input_scattered:
|
|
logging.info(
|
|
"attn_tp_input_scattered is not enabled while other conditions are not met"
|
|
)
|
|
else:
|
|
logging.info("attn_tp_input_scattered is enabled")
|
|
|
|
def use_input_scattered(self, forward_batch: ForwardBatch):
|
|
return (
|
|
self.allow_input_scattered
|
|
and forward_batch.forward_mode.is_extend()
|
|
and not forward_batch.forward_mode.is_target_verify()
|
|
and forward_batch.input_ids is not None
|
|
and not forward_batch.can_run_tbo
|
|
)
|
|
|
|
@property
|
|
def input_scattered(self):
|
|
return get_forward().attn_input_scattered
|
|
|
|
def set_attn_inputs(self, attn_inputs: AttentionInputs):
|
|
get_forward().set("attn_inputs", attn_inputs)
|
|
|
|
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
|
|
attn_inputs = get_forward().attn_inputs
|
|
if attn_inputs is not None:
|
|
attn_inputs.hidden_states_local = hidden_states
|
|
|
|
def fetch_qkv_latent(self):
|
|
attn_inputs = get_forward().attn_inputs
|
|
assert attn_inputs is not None
|
|
return attn_inputs.fetch_qkv_latent()
|
|
|
|
def fetch_hidden_states(self):
|
|
attn_inputs = get_forward().attn_inputs
|
|
assert attn_inputs is not None
|
|
return attn_inputs.fetch_hidden_states()
|
|
|
|
def clear_attn_inputs(self) -> None:
|
|
get_forward().set("attn_inputs", None)
|
|
|
|
@contextmanager
|
|
def maybe_input_scattered(self, forward_batch: ForwardBatch):
|
|
flag = self.use_input_scattered(forward_batch)
|
|
forward = get_forward()
|
|
# scoped() also restores when the forward raises — the old in-place
|
|
# swap leaked the flag on exceptions.
|
|
with forward.scoped(attn_input_scattered=flag):
|
|
try:
|
|
yield
|
|
finally:
|
|
forward.set("attn_inputs", None)
|
|
|
|
|
|
ATTN_TP_CONTEXT = AttnTpContext()
|
|
|
|
|
|
def get_attn_tp_context():
|
|
return ATTN_TP_CONTEXT
|
|
|
|
|
|
@dataclass
|
|
class _LayerModeComputationContext:
|
|
num_layers: int
|
|
layer_id: int
|
|
is_layer_sparse: bool
|
|
is_previous_layer_sparse: Optional[bool]
|
|
is_next_layer_sparse: Optional[bool]
|
|
|
|
def previous_layer(self):
|
|
assert self.is_previous_layer_sparse is not None
|
|
return _LayerModeComputationContext(
|
|
num_layers=self.num_layers,
|
|
layer_id=self.layer_id - 1,
|
|
is_layer_sparse=self.is_previous_layer_sparse,
|
|
is_previous_layer_sparse=None,
|
|
is_next_layer_sparse=self.is_layer_sparse,
|
|
)
|
|
|
|
|
|
def sparse_mlp_scatter_mode() -> ScatterMode:
|
|
"""SCATTERED hands a sparse MLP this rank's own token shard; FULL and
|
|
MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group."""
|
|
if (
|
|
# Token dispatch/combine will be handled outside of LayerCommunicator for these modes.
|
|
not get_moe_a2a_backend().is_none()
|
|
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
|
or enable_dwdp()
|
|
):
|
|
return ScatterMode.SCATTERED
|
|
# DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL.
|
|
if is_enable_moe_cp_allgather() and not (
|
|
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
|
):
|
|
return ScatterMode.MOE_FULL
|
|
return ScatterMode.FULL
|
|
|
|
|
|
@dataclass
|
|
class LayerScatterModes:
|
|
layer_input_mode: ScatterMode
|
|
attn_mode: ScatterMode
|
|
# Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed
|
|
mlp_mode: ScatterMode
|
|
middle_residual_mode: ScatterMode
|
|
layer_output_mode: ScatterMode
|
|
is_layer_sparse: bool = False
|
|
|
|
@classmethod
|
|
def init_new(cls, **kwargs):
|
|
context = _LayerModeComputationContext(**kwargs)
|
|
return cls(
|
|
layer_input_mode=cls._compute_layer_input_mode(context),
|
|
attn_mode=ScatterMode.TP_ATTN_FULL,
|
|
mlp_mode=cls._compute_mlp_mode(context),
|
|
middle_residual_mode=cls._compute_middle_residual_mode(context),
|
|
layer_output_mode=cls._compute_layer_output_mode(context),
|
|
is_layer_sparse=context.is_layer_sparse,
|
|
)
|
|
|
|
@classmethod
|
|
def _compute_layer_input_mode(cls, context: _LayerModeComputationContext):
|
|
if context.layer_id == 0:
|
|
return ScatterMode.model_input_output()
|
|
return cls._compute_layer_output_mode(context.previous_layer())
|
|
|
|
@classmethod
|
|
def _compute_mlp_mode(cls, context: _LayerModeComputationContext):
|
|
if context.is_layer_sparse:
|
|
return sparse_mlp_scatter_mode()
|
|
else:
|
|
if enable_moe_dense_fully_dp():
|
|
return ScatterMode.SCATTERED
|
|
# A TP-sharded dense MLP reduces over the whole TP group, which spans
|
|
# every CP rank; a CP-sharded prefill must gather tokens across CP
|
|
# first or the all-reduce sums different tokens' partial outputs.
|
|
# MLA/DSA CP models do this in DSACPLayerCommunicator instead.
|
|
if _generic_prefill_cp_shards_tokens() and not (
|
|
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
|
):
|
|
return ScatterMode.MOE_FULL
|
|
return ScatterMode.FULL
|
|
|
|
@classmethod
|
|
def _should_gather_for_tbo(cls, context: _LayerModeComputationContext):
|
|
return (
|
|
not context.is_layer_sparse
|
|
and context.is_next_layer_sparse
|
|
and enable_moe_dense_fully_dp()
|
|
and get_exec().overlap.enable_two_batch_overlap
|
|
)
|
|
|
|
@classmethod
|
|
def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext):
|
|
mlp_mode = cls._compute_mlp_mode(context)
|
|
if mlp_mode == ScatterMode.SCATTERED:
|
|
return ScatterMode.SCATTERED
|
|
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def _compute_layer_output_mode(cls, context: _LayerModeComputationContext):
|
|
mlp_mode = cls._compute_mlp_mode(context)
|
|
if context.layer_id == context.num_layers - 1:
|
|
return ScatterMode.model_input_output()
|
|
if mlp_mode == ScatterMode.SCATTERED:
|
|
if cls._should_gather_for_tbo(context):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
return ScatterMode.SCATTERED
|
|
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
raise NotImplementedError
|
|
|
|
|
|
def enable_moe_dense_fully_dp():
|
|
return get_parallel().moe_dense_tp_size == 1
|
|
|
|
|
|
def _generic_prefill_cp_shards_tokens() -> bool:
|
|
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
|
|
parallel = get_parallel()
|
|
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
|
|
|
|
|
|
def enable_dwdp():
|
|
return get_parallel().dwdp_size > 1
|
|
|
|
|
|
def tp_reduce_scatter(
|
|
hidden_states: torch.Tensor,
|
|
residual: Optional[torch.Tensor],
|
|
context: "CommunicateContext",
|
|
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
|
"""Module-level so MHC communicators can reuse it without a
|
|
``LayerCommunicator`` instance."""
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, hidden_states
|
|
assert hidden_states.shape[0] % context.tp_size == 0, (
|
|
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
|
|
)
|
|
local_tokens = hidden_states.shape[0] // context.tp_size
|
|
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
|
|
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
|
|
if residual is not None:
|
|
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
|
return output, residual
|
|
|
|
|
|
def _update_and_read_residual_plain(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
if residual is None:
|
|
return norm(hidden_states), hidden_states
|
|
return norm(hidden_states, residual, post_residual_addition)
|
|
|
|
|
|
def _update_and_read_residual_aiter_mxfp4(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
# post_residual_addition is not applied on this path.
|
|
output, *_, residual_out = fused_rms_mxfp4_quant(
|
|
hidden_states,
|
|
norm.weight,
|
|
norm.variance_epsilon,
|
|
None,
|
|
None,
|
|
None,
|
|
residual,
|
|
)
|
|
return output, hidden_states if residual is None else residual_out
|
|
|
|
|
|
def _update_and_read_residual_aiter_fp8_group(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
"""aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the
|
|
unquantized bf16 output rides along as a third element, so the DSA indexer
|
|
can skip dequantizing. post_residual_addition is not applied on this path."""
|
|
needs_bf16 = get_attn_tp_context().is_dsa
|
|
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
|
|
hidden_states,
|
|
norm.weight,
|
|
norm.variance_epsilon,
|
|
inp2=None,
|
|
inp2_weight=None,
|
|
inp2_epsilon=None,
|
|
group_size=128,
|
|
dtype_quant=torch.float8_e4m3fn,
|
|
res1=residual,
|
|
output_unquantized_inp1=needs_bf16,
|
|
transpose_scale=False,
|
|
)
|
|
if _use_aiter_bpreshuffle_gfx95:
|
|
output = materialize_bpreshuffle_fp8_scale_tuple(output)
|
|
if needs_bf16:
|
|
output = (output[0], output[1], unquantized)
|
|
return output, hidden_states if residual is None else residual_out
|
|
|
|
|
|
def _update_and_read_residual_aiter_fp8_per_token(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
if residual is None:
|
|
output = _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states, norm.weight.data, norm.variance_epsilon
|
|
)
|
|
return output, hidden_states
|
|
if post_residual_addition is not None:
|
|
residual = residual + post_residual_addition
|
|
return _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual
|
|
)
|
|
|
|
|
|
def _attn_input_update_and_read_residual(quant_format: str):
|
|
"""Add the previous layer's output to the residual and read the attention
|
|
input from it: the input norm, fused with the quantization this format
|
|
wants. Without a residual (the first layer, or one already folded into
|
|
hidden_states) the input itself becomes the residual."""
|
|
if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format:
|
|
return _update_and_read_residual_aiter_mxfp4
|
|
if _use_aiter and _is_gfx95_supported and quant_format == "fp8":
|
|
return _update_and_read_residual_aiter_fp8_group
|
|
if _use_aiter and quant_format == "fp8_per_token":
|
|
return _update_and_read_residual_aiter_fp8_per_token
|
|
return _update_and_read_residual_plain
|
|
|
|
|
|
class LayerCommunicator:
|
|
def __init__(
|
|
self,
|
|
layer_scatter_modes: LayerScatterModes,
|
|
input_layernorm: torch.nn.Module,
|
|
post_attention_layernorm: torch.nn.Module,
|
|
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
|
|
allow_reduce_scatter: bool = False,
|
|
is_last_layer: bool = False,
|
|
qkv_latent_func: Optional[Callable] = None,
|
|
force_layernorm_before_dp_gather: bool = False,
|
|
enable_fused_ar_quant: bool = False,
|
|
fused_ar_quant_keep_bf16: bool = False,
|
|
_is_sp_variant: bool = False,
|
|
):
|
|
self.layer_scatter_modes = layer_scatter_modes
|
|
self.input_layernorm = input_layernorm
|
|
self.post_attention_layernorm = post_attention_layernorm
|
|
self.allow_reduce_scatter = allow_reduce_scatter
|
|
self.is_last_layer = is_last_layer
|
|
self.qkv_latent_func = qkv_latent_func
|
|
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
|
|
self.enable_fused_ar_quant = enable_fused_ar_quant
|
|
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
|
|
|
self._context = CommunicateContext.init_new()
|
|
self._context.force_layernorm_before_dp_gather = (
|
|
force_layernorm_before_dp_gather
|
|
)
|
|
self._post_init_communicate()
|
|
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
|
get_spec().speculative_algorithm
|
|
)
|
|
|
|
# Under LayerNorm SP the norm/residual run on the sequence shard with no
|
|
# collectives, so delegate to an all-SCATTERED sibling while the region is
|
|
# active. _is_sp_variant stops the sibling from building its own.
|
|
self._sp_variant: Optional[LayerCommunicator] = None
|
|
if not _is_sp_variant and layernorm_sp.layernorm_sp_enabled():
|
|
self._sp_variant = LayerCommunicator(
|
|
layer_scatter_modes=LayerScatterModes(
|
|
layer_input_mode=ScatterMode.SCATTERED,
|
|
attn_mode=ScatterMode.SCATTERED,
|
|
mlp_mode=ScatterMode.SCATTERED,
|
|
middle_residual_mode=ScatterMode.SCATTERED,
|
|
layer_output_mode=ScatterMode.SCATTERED,
|
|
),
|
|
input_layernorm=input_layernorm,
|
|
post_attention_layernorm=post_attention_layernorm,
|
|
allow_reduce_scatter=allow_reduce_scatter,
|
|
is_last_layer=is_last_layer,
|
|
qkv_latent_func=qkv_latent_func,
|
|
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
|
enable_fused_ar_quant=enable_fused_ar_quant,
|
|
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
|
|
_is_sp_variant=True,
|
|
)
|
|
|
|
def _post_init_communicate(self):
|
|
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
|
input_mode=self.layer_scatter_modes.layer_input_mode,
|
|
output_mode=self.layer_scatter_modes.attn_mode,
|
|
context=self._context,
|
|
)
|
|
self._communicate_with_all_reduce_and_layer_norm_fn = (
|
|
CommunicateWithAllReduceAndLayerNormFn.get_fn(
|
|
hidden_states_input_mode=self.layer_scatter_modes.attn_mode,
|
|
residual_input_mode=self.layer_scatter_modes.layer_input_mode,
|
|
hidden_states_output_mode=self.layer_scatter_modes.mlp_mode,
|
|
residual_output_mode=self.layer_scatter_modes.middle_residual_mode,
|
|
context=self._context,
|
|
)
|
|
)
|
|
self._communicate_summable_tensor_pair_fn = (
|
|
CommunicateSummableTensorPairFn.get_fn(
|
|
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
|
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
|
output_mode=self.layer_scatter_modes.layer_output_mode,
|
|
context=self._context,
|
|
)
|
|
)
|
|
|
|
def prepare_attn_and_capture_last_layer_outputs(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
|
post_residual_addition: Optional[torch.Tensor] = None,
|
|
quant_format: str = "",
|
|
):
|
|
hidden_states, residual = self.prepare_attn(
|
|
hidden_states,
|
|
residual,
|
|
forward_batch,
|
|
quant_format=quant_format,
|
|
post_residual_addition=post_residual_addition,
|
|
)
|
|
if captured_last_layer_outputs is not None:
|
|
gathered_last_layer_output = self._communicate_simple_fn(
|
|
hidden_states=residual,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
)
|
|
if (
|
|
gathered_last_layer_output is residual
|
|
# An accumulator that copies on append already holds a snapshot.
|
|
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
|
|
and not self._post_attn_residual_is_read_only(residual)
|
|
):
|
|
gathered_last_layer_output = residual.clone()
|
|
captured_last_layer_outputs.append(gathered_last_layer_output)
|
|
return hidden_states, residual
|
|
|
|
def _post_attn_residual_is_read_only(self, residual: torch.Tensor) -> bool:
|
|
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
|
|
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
|
|
|
|
Only the flashinfer all-reduce-fusion path writes a fresh ``residual_out``
|
|
(see ``flashinfer_allreduce_residual_rmsnorm``); the aiter fused kernel and
|
|
every plain norm fold into ``residual`` in place. That path is reachable
|
|
only from the ``_gather_*`` communicate-fns, and only when they fall past
|
|
their input-scattered branch.
|
|
"""
|
|
norm_fn = getattr(
|
|
self._communicate_with_all_reduce_and_layer_norm_fn,
|
|
"func",
|
|
self._communicate_with_all_reduce_and_layer_norm_fn,
|
|
)
|
|
uses_gather_norm = norm_fn in (
|
|
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
|
|
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
|
|
)
|
|
return (
|
|
uses_gather_norm
|
|
and not get_attn_tp_context().input_scattered
|
|
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
|
)
|
|
|
|
def prepare_attn(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
quant_format: str = "",
|
|
post_residual_addition: Optional[torch.Tensor] = None,
|
|
):
|
|
# residual is None marks the first decoder layer, where the SP region
|
|
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
|
|
# the next one.
|
|
if self._sp_variant is not None:
|
|
if residual is None:
|
|
get_forward().set(
|
|
"sp_active", forward_batch.forward_mode == ForwardMode.EXTEND
|
|
)
|
|
if get_forward().sp_active:
|
|
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
|
if get_forward().sp_active:
|
|
return self._sp_variant.prepare_attn(
|
|
hidden_states,
|
|
residual,
|
|
forward_batch,
|
|
quant_format,
|
|
post_residual_addition,
|
|
)
|
|
if get_attn_tp_context().input_scattered:
|
|
hidden_states, residual = self._tp_reduce_scatter(
|
|
hidden_states,
|
|
residual,
|
|
)
|
|
if hidden_states.shape[0] == 0:
|
|
residual = hidden_states
|
|
elif (
|
|
residual is not None
|
|
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
|
and hidden_states._sglang_needs_allreduce_fusion
|
|
):
|
|
hidden_states, residual = self._reduce_output_and_update_and_read_residual(
|
|
hidden_states, residual, forward_batch
|
|
)
|
|
else:
|
|
hidden_states, residual = _attn_input_update_and_read_residual(
|
|
quant_format
|
|
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
|
|
|
return self._finish_prepare_attn(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
)
|
|
|
|
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
|
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
|
hidden_states = self._communicate_simple_fn(
|
|
hidden_states=hidden_states,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
)
|
|
if self.qkv_latent_func is not None:
|
|
attn_inputs = AttentionInputs(
|
|
hidden_states, forward_batch, self.qkv_latent_func
|
|
)
|
|
get_attn_tp_context().set_attn_inputs(attn_inputs)
|
|
return hidden_states, residual
|
|
|
|
def _reduce_output_and_update_and_read_residual(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""Complete the sum the previous layer left, add it to the residual and
|
|
apply the input norm:
|
|
fused into one kernel when it takes this batch, else as an all-reduce
|
|
followed by the norm. Neither applies ``quant_format`` or
|
|
``post_residual_addition``."""
|
|
if (
|
|
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
|
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
|
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
|
if (
|
|
self.enable_fused_ar_quant
|
|
and _use_aiter
|
|
and hasattr(
|
|
self.input_layernorm,
|
|
"forward_with_allreduce_fusion_quant_per_group",
|
|
)
|
|
):
|
|
# Falls back to AR+RMSNorm + separate quant internally when the
|
|
# fully-fused kernel cannot service the shape.
|
|
quant_result = (
|
|
self.input_layernorm.forward_with_allreduce_fusion_quant_per_group(
|
|
hidden_states,
|
|
residual,
|
|
use_attn_tp_group=False,
|
|
keep_bf16=self.fused_ar_quant_keep_bf16,
|
|
)
|
|
)
|
|
if quant_result is not None:
|
|
return quant_result
|
|
return self.input_layernorm.forward_with_allreduce_fusion(
|
|
hidden_states, residual, use_attn_tp_group=False
|
|
)
|
|
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
|
return self.input_layernorm(hidden_states, residual)
|
|
|
|
def _tp_reduce_scatter(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
return tp_reduce_scatter(hidden_states, residual, self._context)
|
|
|
|
def prepare_mlp(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
cache=None,
|
|
):
|
|
if self._sp_variant is not None and get_forward().sp_active:
|
|
return self._sp_variant.prepare_mlp(
|
|
hidden_states, residual, forward_batch, cache
|
|
)
|
|
if cache is not None:
|
|
self._context.cache = cache
|
|
|
|
return self._communicate_with_all_reduce_and_layer_norm_fn(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
layernorm=self.post_attention_layernorm,
|
|
context=self._context,
|
|
)
|
|
|
|
def maybe_prefetch_next_full_attention_kv(
|
|
self,
|
|
forward_batch: ForwardBatch,
|
|
next_full_attention_layer_id: Optional[int],
|
|
) -> None:
|
|
return
|
|
|
|
def postprocess_layer(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
):
|
|
if self._sp_variant is not None and get_forward().sp_active:
|
|
return self._sp_variant.postprocess_layer(
|
|
hidden_states, residual, forward_batch
|
|
)
|
|
return self._communicate_summable_tensor_pair_fn(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
allow_reduce_scatter=self.allow_reduce_scatter,
|
|
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
|
)
|
|
|
|
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
|
"""Decide once how this layer's FFN output reduction completes. Use the
|
|
result as a context manager around the FFN call, then call ``finish``."""
|
|
return FfnExit(self, forward_batch)
|
|
|
|
def finish_layer_stack(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: Optional[torch.Tensor],
|
|
forward_batch: ForwardBatch,
|
|
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
|
"""Complete what this layer left for a next layer. Call it on the last
|
|
layer of this rank before its output reaches the final norm, the next
|
|
pipeline rank, or any other consumer outside the layers."""
|
|
return complete_deferred_allreduce(hidden_states), residual
|
|
|
|
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
|
if not self.allow_reduce_scatter:
|
|
return False
|
|
if (
|
|
self._communicate_summable_tensor_pair_fn
|
|
is CommunicateSummableTensorPairFn._scatter_hidden_states
|
|
):
|
|
if should_use_dp_reduce_scatterv():
|
|
return True
|
|
if (
|
|
forward_batch.dp_padding_mode.is_max_len()
|
|
and can_use_dp_reduce_scatter()
|
|
):
|
|
return True
|
|
# Prefill CP predicates must stay out of decode graph capture.
|
|
if forward_batch.forward_mode.is_context_parallel_extend() and (
|
|
dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch)
|
|
):
|
|
return True
|
|
if get_attn_tp_context().input_scattered and not self.is_last_layer:
|
|
return True
|
|
return False
|
|
|
|
def should_defer_moe_finalize(
|
|
self, forward_batch: ForwardBatch, m: int | None = None
|
|
) -> bool:
|
|
"""Whether the MoE may hand an unfinalized output to the next layer."""
|
|
return False
|
|
|
|
# NOTE: This function will cause torch recompilation
|
|
def should_fuse_mlp_allreduce_with_next_layer(
|
|
self, forward_batch: ForwardBatch
|
|
) -> bool:
|
|
# When MOE_FULL is active (moe_cp allgather), fusion must be disabled because
|
|
# the fusion path skips postprocess_layer which contains the moe_cp scatter.
|
|
# Without scatter, hidden_states remain at MOE_FULL size while residual is at
|
|
# TP_ATTN_FULL size, causing a shape mismatch.
|
|
if (
|
|
is_enable_moe_cp_allgather()
|
|
or self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL
|
|
):
|
|
return False
|
|
|
|
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
|
# two disjoint groups; post_experts_all_reduce() merges them into one
|
|
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
|
|
# When merging is blocked, no single group covers both, so fusion stays off.
|
|
parallel = get_parallel()
|
|
if (
|
|
parallel.moe_ep_size > 1
|
|
and parallel.moe_tp_size > 1
|
|
and not can_merge_post_experts_all_reduce()
|
|
):
|
|
return False
|
|
|
|
if (
|
|
is_dp_attention_enabled()
|
|
and self._speculative_algo is not None
|
|
and self._speculative_algo.is_eagle()
|
|
):
|
|
return False
|
|
|
|
if get_attn_tp_context().input_scattered:
|
|
return False
|
|
|
|
batch_size = (
|
|
forward_batch.input_ids.shape[0]
|
|
if hasattr(forward_batch, "input_ids")
|
|
else 0
|
|
)
|
|
|
|
# When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP
|
|
# all-reduce, so there is nothing to fuse with the next layer.
|
|
if self.layer_scatter_modes.mlp_mode == ScatterMode.SCATTERED:
|
|
return False
|
|
|
|
return (
|
|
(
|
|
apply_flashinfer_allreduce_fusion(batch_size)
|
|
or (
|
|
_use_aiter
|
|
and batch_size > 0
|
|
and get_parallel().tp_size != 6
|
|
and not is_dp_attention_enabled()
|
|
and get_moe_a2a_backend().is_none()
|
|
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
|
)
|
|
)
|
|
and (not self.is_last_layer)
|
|
and (self._context.tp_size > 1)
|
|
)
|
|
|
|
|
|
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
|
|
_SCATTER_MODE_SHARDED_AXES = {
|
|
ScatterMode.SCATTERED: (
|
|
TokenAxis.ATTN_DP,
|
|
TokenAxis.ATTN_CP,
|
|
TokenAxis.ATTN_TP_SCATTER,
|
|
),
|
|
ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP),
|
|
ScatterMode.FULL: (TokenAxis.ATTN_CP,),
|
|
ScatterMode.MOE_FULL: (),
|
|
}
|
|
|
|
|
|
def scatter_mode_layouts(
|
|
*, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int
|
|
) -> Dict[ScatterMode, Layout]:
|
|
axis_sizes = {
|
|
TokenAxis.ATTN_DP: attn_dp_size,
|
|
TokenAxis.ATTN_CP: attn_cp_size,
|
|
TokenAxis.ATTN_TP_SCATTER: attn_tp_size,
|
|
}
|
|
return {
|
|
mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes)
|
|
for mode, axes in _SCATTER_MODE_SHARDED_AXES.items()
|
|
}
|
|
|
|
|
|
class FfnExit:
|
|
"""One FFN's reduction decision. Inside the ``with`` block it is published as
|
|
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on
|
|
``get_forward()``."""
|
|
|
|
__slots__ = (
|
|
"communicator",
|
|
"forward_batch",
|
|
"defer_moe_finalize",
|
|
"fuse_mlp_allreduce",
|
|
"mlp_reduce_scatter",
|
|
"_scope",
|
|
)
|
|
|
|
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
|
self.communicator = communicator
|
|
self.forward_batch = forward_batch
|
|
self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch)
|
|
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
|
self.fuse_mlp_allreduce = (
|
|
self.defer_moe_finalize
|
|
or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
|
)
|
|
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
|
|
self._scope = get_forward().scoped(
|
|
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
|
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
|
defer_moe_finalize=self.defer_moe_finalize,
|
|
)
|
|
|
|
def __enter__(self) -> "FfnExit":
|
|
self._scope.__enter__()
|
|
return self
|
|
|
|
def __exit__(self, *exc_info):
|
|
return self._scope.__exit__(*exc_info)
|
|
|
|
def finish(
|
|
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""Leave the reduction to the next layer's input norm, or postprocess."""
|
|
if not isinstance(hidden_states, torch.Tensor):
|
|
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
|
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
|
return hidden_states, residual
|
|
if self.fuse_mlp_allreduce:
|
|
hidden_states._sglang_needs_allreduce_fusion = True
|
|
return hidden_states, residual
|
|
return self.communicator.postprocess_layer(
|
|
hidden_states, residual, self.forward_batch
|
|
)
|
|
|
|
|
|
def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor:
|
|
"""Run the all-reduce a layer left for the next layer's input norm."""
|
|
if (
|
|
hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
|
and hidden_states._sglang_needs_allreduce_fusion
|
|
):
|
|
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
|
hidden_states._sglang_needs_allreduce_fusion = False
|
|
return hidden_states
|
|
|
|
|
|
@dataclass
|
|
class CommunicateContext:
|
|
process_group_sizes: Dict[ScatterMode, int]
|
|
attn_tp_rank: int
|
|
attn_tp_size: int
|
|
attn_dp_size: int
|
|
attn_cp_rank: int
|
|
attn_cp_size: int
|
|
tp_size: int
|
|
cache = None
|
|
tp_rank: int
|
|
force_layernorm_before_dp_gather: bool = False
|
|
|
|
def is_same_group_size(self, a: ScatterMode, b: ScatterMode):
|
|
return self.process_group_sizes[a] == self.process_group_sizes[b]
|
|
|
|
@cached_property
|
|
def layouts(self) -> Dict[ScatterMode, Layout]:
|
|
return scatter_mode_layouts(
|
|
attn_dp_size=self.attn_dp_size,
|
|
attn_cp_size=self.attn_cp_size,
|
|
attn_tp_size=self.attn_tp_size,
|
|
)
|
|
|
|
def is_same_layout(self, a: ScatterMode, b: ScatterMode):
|
|
return self.layouts[a] == self.layouts[b]
|
|
|
|
@classmethod
|
|
def init_new(cls):
|
|
attn_tp_rank = get_parallel().attn_tp_rank
|
|
attn_tp_size = get_parallel().attn_tp_size
|
|
attn_dp_size = get_parallel().attn_dp_size
|
|
attn_cp_size = get_parallel().attn_cp_size
|
|
attn_cp_rank = get_parallel().attn_cp_rank
|
|
tp_size = get_parallel().tp_size
|
|
tp_rank = get_parallel().tp_rank
|
|
moe_cp_size = get_moe_cp_size()
|
|
process_group_sizes = {
|
|
ScatterMode.SCATTERED: 1,
|
|
ScatterMode.TP_ATTN_FULL: attn_tp_size,
|
|
# TODO: support --moe-dense-tp-size > 1
|
|
# With context parallel enabled, we should exclude
|
|
# the attn_cp_size from the total tp_size
|
|
ScatterMode.FULL: tp_size // attn_cp_size,
|
|
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
|
|
}
|
|
return cls(
|
|
process_group_sizes=process_group_sizes,
|
|
attn_tp_rank=attn_tp_rank,
|
|
attn_tp_size=attn_tp_size,
|
|
attn_dp_size=attn_dp_size,
|
|
attn_cp_rank=attn_cp_rank,
|
|
attn_cp_size=attn_cp_size,
|
|
tp_size=tp_size,
|
|
tp_rank=tp_rank,
|
|
)
|
|
|
|
|
|
class CommunicateSimpleFn:
|
|
@staticmethod
|
|
def get_fn(
|
|
input_mode: ScatterMode,
|
|
output_mode: ScatterMode,
|
|
context: CommunicateContext,
|
|
):
|
|
if context.is_same_layout(input_mode, output_mode):
|
|
return CommunicateSimpleFn._trivial
|
|
|
|
if (input_mode == ScatterMode.SCATTERED) and (
|
|
output_mode == ScatterMode.TP_ATTN_FULL
|
|
):
|
|
if _use_ag_after_qlora:
|
|
return CommunicateSimpleFn._trivial
|
|
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
|
|
|
raise NotImplementedError(f"{input_mode=} {output_mode=}")
|
|
|
|
@staticmethod
|
|
def _trivial(
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
) -> torch.Tensor:
|
|
return hidden_states
|
|
|
|
@staticmethod
|
|
def _scattered_to_tp_attn_full(
|
|
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
|
if isinstance(hidden_states, tuple):
|
|
gathered_hidden_states = []
|
|
for local_hidden_states in hidden_states:
|
|
with use_symmetric_memory(
|
|
get_parallel().tp_group,
|
|
disabled=not is_allocation_symmetric(),
|
|
):
|
|
output = torch.empty(
|
|
(
|
|
local_hidden_states.shape[0] * context.attn_tp_size,
|
|
*local_hidden_states.shape[1:],
|
|
),
|
|
dtype=local_hidden_states.dtype,
|
|
device=local_hidden_states.device,
|
|
)
|
|
attn_tp_all_gather_into_tensor(
|
|
output,
|
|
local_hidden_states,
|
|
)
|
|
gathered_hidden_states.append(output)
|
|
return tuple(gathered_hidden_states)
|
|
|
|
hidden_states, local_hidden_states = (
|
|
get_local_dp_buffer(get_parallel().attn_tp_group),
|
|
hidden_states,
|
|
)
|
|
attn_tp_all_gather_into_tensor(
|
|
hidden_states,
|
|
local_hidden_states,
|
|
)
|
|
return hidden_states
|
|
|
|
|
|
class CommunicateWithAllReduceAndLayerNormFn:
|
|
"""Besides communication, needs to
|
|
1. All reduce in tp_attn_group on hidden_states
|
|
2. Apply layer norm
|
|
"""
|
|
|
|
@staticmethod
|
|
def get_fn(
|
|
hidden_states_input_mode: ScatterMode,
|
|
residual_input_mode: ScatterMode,
|
|
hidden_states_output_mode: ScatterMode,
|
|
residual_output_mode: ScatterMode,
|
|
context: CommunicateContext,
|
|
):
|
|
|
|
if (
|
|
context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode)
|
|
and context.is_same_layout(residual_input_mode, residual_output_mode)
|
|
and context.attn_tp_size == 1
|
|
):
|
|
return CommunicateWithAllReduceAndLayerNormFn._simple
|
|
|
|
if (
|
|
hidden_states_input_mode == ScatterMode.SCATTERED
|
|
and residual_input_mode == ScatterMode.SCATTERED
|
|
and hidden_states_output_mode == ScatterMode.SCATTERED
|
|
and residual_output_mode == ScatterMode.SCATTERED
|
|
):
|
|
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
|
|
# activations stay sequence-sharded across the attn->mlp boundary, so
|
|
# there is nothing to gather or scatter here -- just the residual add
|
|
# plus LayerNorm on the local shard. The row-parallel o_proj already
|
|
# issued the reduce-scatter (g-bar) that the all-reduce would have
|
|
# done, and the g all-gather is fused into the next column-parallel
|
|
# linear. Distinct from the branch above because under pure TP
|
|
# attn_tp_size == tp_size > 1, so that gate does not fire.
|
|
return CommunicateWithAllReduceAndLayerNormFn._simple
|
|
|
|
if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and (
|
|
residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL)
|
|
):
|
|
fn = {
|
|
(
|
|
ScatterMode.FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual,
|
|
(
|
|
ScatterMode.MOE_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe,
|
|
(
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.SCATTERED,
|
|
): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual,
|
|
}.get((hidden_states_output_mode, residual_output_mode))
|
|
if fn is not None:
|
|
return partial(fn, residual_input_mode=residual_input_mode)
|
|
|
|
if (
|
|
hidden_states_output_mode == ScatterMode.TP_ATTN_FULL
|
|
and residual_output_mode == ScatterMode.TP_ATTN_FULL
|
|
and context.attn_tp_size > 1
|
|
):
|
|
# Used when the dense MLP is tensor-parallelized along the
|
|
# attention TP group (``moe_dense_tp_size > 1``): hidden states
|
|
# need an all-reduce inside the attention TP group before the
|
|
# next layernorm, while staying in TP_ATTN_FULL on both sides.
|
|
return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm
|
|
|
|
raise NotImplementedError(
|
|
f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}"
|
|
)
|
|
|
|
@staticmethod
|
|
def _simple(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
# TODO move these `if shape != 0` into LayerNorm itself
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _tp_attn_all_reduce_and_layernorm(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
"""All-reduce hidden states inside the attention TP group, then layernorm.
|
|
|
|
Used when the dense MLP shares the attention TP group
|
|
(``moe_dense_tp_size > 1``): both hidden states and residual stay in
|
|
``TP_ATTN_FULL`` across the boundary.
|
|
"""
|
|
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _gather_hidden_states_and_residual(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
residual_input_mode,
|
|
):
|
|
if get_attn_tp_context().input_scattered:
|
|
return CommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual(
|
|
hidden_states,
|
|
residual,
|
|
layernorm,
|
|
context,
|
|
)
|
|
|
|
if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1:
|
|
residual, local_residual = (
|
|
get_local_dp_buffer(get_parallel().attn_tp_group),
|
|
residual,
|
|
)
|
|
attn_tp_all_gather_into_tensor(residual, local_residual)
|
|
if context.attn_dp_size != 1:
|
|
use_layer_norm_before_gather = (
|
|
context.force_layernorm_before_dp_gather or context.attn_tp_size == 1
|
|
)
|
|
if use_layer_norm_before_gather and hidden_states.shape[0] != 0:
|
|
if context.attn_tp_size > 1:
|
|
hidden_states = attention_tensor_model_parallel_all_reduce(
|
|
hidden_states
|
|
)
|
|
with use_symmetric_memory(
|
|
get_parallel().tp_group,
|
|
disabled=not is_allocation_symmetric(),
|
|
):
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
elif context.attn_tp_rank == 0:
|
|
hidden_states += residual
|
|
|
|
hidden_states, local_hidden_states = (
|
|
get_global_dp_buffer(get_parallel().tp_group),
|
|
hidden_states,
|
|
)
|
|
if use_layer_norm_before_gather:
|
|
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
|
|
else:
|
|
dp_gather_partial(hidden_states, local_hidden_states, forward_batch)
|
|
|
|
if not use_layer_norm_before_gather:
|
|
dp_scatter(residual, hidden_states, forward_batch)
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states = layernorm(hidden_states)
|
|
else:
|
|
handled = False
|
|
if (
|
|
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
|
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
|
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
|
|
hidden_states, residual = layernorm.forward_with_allreduce_fusion(
|
|
hidden_states, residual, use_attn_tp_group=True
|
|
)
|
|
handled = True
|
|
|
|
if not handled:
|
|
quantize_communications = (
|
|
not forward_batch.forward_mode.is_decode_or_idle()
|
|
and get_exec().comm.enable_quant_communications
|
|
)
|
|
if quantize_communications:
|
|
hidden_states = attention_tensor_model_parallel_quant_all_reduce(
|
|
hidden_states
|
|
)
|
|
else:
|
|
hidden_states = attention_tensor_model_parallel_all_reduce(
|
|
hidden_states
|
|
)
|
|
if _is_npu and context.cache is not None:
|
|
_ = prepare_weight_cache(hidden_states, context.cache)
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _scatter_hidden_states_and_residual(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
residual_input_mode,
|
|
):
|
|
input_hidden_states = hidden_states
|
|
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
|
context.attn_tp_rank
|
|
]
|
|
attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states)
|
|
if residual_input_mode == ScatterMode.TP_ATTN_FULL:
|
|
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _tp_all_reduce_with_scattered_residual(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, hidden_states
|
|
|
|
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
|
scattered_states += residual
|
|
residual = tensor_model_parallel_all_reduce(hidden_states)
|
|
hidden_states = layernorm(residual)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _gather_hidden_states_and_residual_moe(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
residual_input_mode,
|
|
):
|
|
"""Allgather tokens for MoE when moe_dp_size < attn_cp_size.
|
|
|
|
Steps:
|
|
1. Standard attn-TP all-reduce + optional DP allgather + layernorm (same as
|
|
_gather_hidden_states_and_residual for the dp>1 case, or simple all-reduce
|
|
+ layernorm for dp==1).
|
|
2. moe_cp allgather: gather tokens from cp_per_moe CP ranks so each rank holds
|
|
all tokens for its MoE group.
|
|
|
|
Residual is left at TP_ATTN_FULL throughout.
|
|
"""
|
|
# Early return on empty tensor is safe for MOE_CP because:
|
|
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
|
# so no rank hits this path while others proceed to the allgather.
|
|
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
|
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, residual
|
|
|
|
# Step 1: Standard all-reduce/DP-allgather + layernorm (reuse existing logic).
|
|
hidden_states, residual = (
|
|
CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
layernorm=layernorm,
|
|
context=context,
|
|
residual_input_mode=residual_input_mode,
|
|
)
|
|
)
|
|
|
|
# Step 2: moe_cp allgather — gather across cp_per_moe CP ranks.
|
|
# Only active during prefill (context-parallel extend); decode keeps existing path.
|
|
moe_cp_size = get_moe_cp_size()
|
|
if (
|
|
moe_cp_size > 1
|
|
and hidden_states.shape[0] > 0
|
|
and forward_batch.forward_mode.is_context_parallel_extend()
|
|
and forward_batch.attn_cp_metadata is not None
|
|
):
|
|
# Zigzag split can produce unequal token counts across CP ranks
|
|
# (when seq_len % (cp_size * 2) != 0). NCCL allgather requires
|
|
# equal input sizes, so pad to the max per-rank token count.
|
|
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
|
|
max_tokens = max(per_rank_tokens)
|
|
pad_size = max_tokens - hidden_states.shape[0]
|
|
if pad_size > 0:
|
|
hidden_states = torch.nn.functional.pad(
|
|
hidden_states, [0, 0, 0, pad_size]
|
|
)
|
|
|
|
output = torch.empty(
|
|
(max_tokens * moe_cp_size, hidden_states.shape[1]),
|
|
dtype=hidden_states.dtype,
|
|
device=hidden_states.device,
|
|
)
|
|
moe_cp_all_gather_into_tensor(output, hidden_states)
|
|
hidden_states = output
|
|
|
|
return hidden_states, residual
|
|
|
|
|
|
class CommunicateSummableTensorPairFn:
|
|
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
|
|
|
|
@classmethod
|
|
def execute(
|
|
cls,
|
|
hidden_states_input_mode,
|
|
residual_input_mode,
|
|
output_mode,
|
|
context,
|
|
**kwargs,
|
|
):
|
|
return cls.get_fn(
|
|
hidden_states_input_mode=hidden_states_input_mode,
|
|
residual_input_mode=residual_input_mode,
|
|
output_mode=output_mode,
|
|
context=context,
|
|
)(context=context, **kwargs)
|
|
|
|
@staticmethod
|
|
def get_fn(
|
|
hidden_states_input_mode: ScatterMode,
|
|
residual_input_mode: ScatterMode,
|
|
output_mode: ScatterMode,
|
|
context: CommunicateContext,
|
|
):
|
|
if context.is_same_layout(
|
|
hidden_states_input_mode, output_mode
|
|
) and context.is_same_layout(residual_input_mode, output_mode):
|
|
return CommunicateSummableTensorPairFn._trivial
|
|
|
|
fn = {
|
|
(
|
|
ScatterMode.FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._scatter_hidden_states,
|
|
(
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._gather,
|
|
(
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.SCATTERED,
|
|
): CommunicateSummableTensorPairFn._scatter,
|
|
(
|
|
ScatterMode.MOE_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
|
}.get((hidden_states_input_mode, residual_input_mode, output_mode))
|
|
if fn is not None:
|
|
return fn
|
|
|
|
raise NotImplementedError(
|
|
f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}"
|
|
)
|
|
|
|
@staticmethod
|
|
def _trivial(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _scatter_hidden_states(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
allow_reduce_scatter: bool = False,
|
|
is_layer_sparse: bool = False,
|
|
):
|
|
if get_parallel().tp_size == get_parallel().attn_dp_size:
|
|
group = get_parallel().tp_group
|
|
else:
|
|
group = get_parallel().attn_tp_group
|
|
hidden_states, global_hidden_states = (
|
|
get_local_dp_buffer(group),
|
|
hidden_states,
|
|
)
|
|
# A MoE block leaves its sum to reduce_scatterv whenever it applies
|
|
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
|
|
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
|
|
if should_use_dp_reduce_scatterv() and (
|
|
allow_reduce_scatter or is_layer_sparse
|
|
):
|
|
get_parallel().tp_group.reduce_scatterv(
|
|
global_hidden_states,
|
|
output=hidden_states,
|
|
sizes=get_dp_global_num_tokens(),
|
|
)
|
|
elif (
|
|
allow_reduce_scatter
|
|
and forward_batch.dp_padding_mode.is_max_len()
|
|
and can_use_dp_reduce_scatter()
|
|
):
|
|
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
|
|
else:
|
|
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _gather(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
hidden_states += residual
|
|
residual = None
|
|
hidden_states, local_hidden_states = (
|
|
get_local_dp_buffer(get_parallel().attn_tp_group),
|
|
hidden_states,
|
|
)
|
|
attn_tp_all_gather_into_tensor(
|
|
hidden_states,
|
|
local_hidden_states,
|
|
)
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _scatter(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
):
|
|
assert residual is None, "not yet handled residual!=None"
|
|
tensor_list = list(hidden_states.tensor_split(context.attn_tp_size))
|
|
hidden_states = tensor_list[context.attn_tp_rank]
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _scatter_hidden_states_moe(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
|
|
|
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
|
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
|
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
|
CP-local portion.
|
|
|
|
If DP>1, further scatter back to the local DP slice.
|
|
"""
|
|
# Only scatter back during prefill; decode was never allgathered so no-op.
|
|
# Safe w.r.t. empty tensors: same reasoning as _gather_hidden_states_and_residual_moe
|
|
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
|
moe_cp_size = get_moe_cp_size()
|
|
if (
|
|
moe_cp_size > 1
|
|
and forward_batch.forward_mode.is_context_parallel_extend()
|
|
and forward_batch.attn_cp_metadata is not None
|
|
):
|
|
moe_cp_rank = get_moe_cp_rank()
|
|
# The allgather was padded to max_tokens_per_rank (equal chunks).
|
|
# Extract this rank's actual (non-padded) tokens from its chunk.
|
|
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
|
|
max_tokens_per_rank = max(per_rank_tokens)
|
|
actual_local_tokens = per_rank_tokens[moe_cp_rank]
|
|
hidden_states = hidden_states.narrow(
|
|
0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens
|
|
).contiguous()
|
|
|
|
# DP scatter (if DP attention is enabled)
|
|
if context.attn_dp_size > 1:
|
|
if get_parallel().tp_size == get_parallel().attn_dp_size:
|
|
group = get_parallel().tp_group
|
|
else:
|
|
group = get_parallel().attn_tp_group
|
|
hidden_states_output, global_hidden_states = (
|
|
get_local_dp_buffer(group),
|
|
hidden_states,
|
|
)
|
|
dp_scatter(hidden_states_output, global_hidden_states, forward_batch)
|
|
hidden_states = hidden_states_output
|
|
|
|
return hidden_states, residual
|