2275 lines
86 KiB
Python
2275 lines
86 KiB
Python
# Copyright 2023-2024 SGLang Team
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
import logging
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from enum import Enum, auto
|
|
from functools import cached_property, partial
|
|
from typing import Callable, Dict, Optional, Tuple, Union
|
|
|
|
import msgspec
|
|
import torch
|
|
|
|
from sglang.srt.distributed import (
|
|
GroupCoordinator,
|
|
attention_tensor_model_parallel_all_reduce,
|
|
attention_tensor_model_parallel_quant_all_reduce,
|
|
tensor_model_parallel_all_reduce,
|
|
)
|
|
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
|
use_symmetric_memory,
|
|
)
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.layers import layernorm_sp
|
|
from sglang.srt.layers.attention.dsa.utils import (
|
|
dsa_use_prefill_cp,
|
|
is_dsa_enable_prefill_cp,
|
|
)
|
|
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
|
from sglang.srt.layers.boundary_layout import (
|
|
DecoderLayerSides,
|
|
Layout,
|
|
StageInput,
|
|
StageOutput,
|
|
SumGroup,
|
|
TokenAxis,
|
|
dense_decoder_layer_sides,
|
|
)
|
|
from sglang.srt.layers.cp.utils import (
|
|
is_mla_cp_active,
|
|
is_mla_cp_enabled,
|
|
)
|
|
from sglang.srt.layers.dp_attention import (
|
|
attn_tp_all_gather_into_tensor,
|
|
attn_tp_reduce_scatter_tensor,
|
|
can_use_dp_reduce_scatter,
|
|
dp_gather_partial,
|
|
dp_gather_replicate,
|
|
dp_reduce_scatter_tensor,
|
|
dp_scatter,
|
|
get_dp_global_num_tokens,
|
|
get_global_dp_buffer,
|
|
get_local_dp_buffer,
|
|
get_moe_cp_rank,
|
|
get_moe_cp_size,
|
|
is_allocation_symmetric,
|
|
is_dp_attention_enabled,
|
|
is_enable_moe_cp_allgather,
|
|
moe_cp_all_gather_into_tensor,
|
|
)
|
|
from sglang.srt.layers.flashinfer_comm_fusion import (
|
|
is_flashinfer_allreduce_unavailable,
|
|
uses_cutedsl_ar_fusion,
|
|
)
|
|
from sglang.srt.layers.moe import (
|
|
can_merge_post_experts_all_reduce,
|
|
get_moe_a2a_backend,
|
|
post_experts_output_is_complete,
|
|
post_experts_reduction_group,
|
|
should_use_dp_reduce_scatterv,
|
|
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
|
)
|
|
from sglang.srt.layers.quantization.fp8_utils import (
|
|
_use_aiter_bpreshuffle_gfx95,
|
|
materialize_bpreshuffle_fp8_scale_tuple,
|
|
)
|
|
from sglang.srt.model_executor.cuda_graph_config import (
|
|
Backend,
|
|
Phase,
|
|
check_cuda_graph_backend,
|
|
)
|
|
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
|
from sglang.srt.runtime_context import (
|
|
LoRABatchLayout,
|
|
get_exec,
|
|
get_forward,
|
|
get_lora,
|
|
get_parallel,
|
|
get_platform,
|
|
get_spec,
|
|
)
|
|
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
|
from sglang.srt.utils import (
|
|
get_bool_env_var,
|
|
is_cuda,
|
|
is_flashinfer_available,
|
|
is_gfx95_supported,
|
|
is_gfx1250_supported,
|
|
is_hip,
|
|
is_npu,
|
|
)
|
|
|
|
_is_cuda = is_cuda()
|
|
_is_flashinfer_available = is_flashinfer_available()
|
|
_is_sm90_supported = _is_cuda and get_platform().is_sm90
|
|
_is_sm100_supported = _is_cuda and get_platform().is_sm100
|
|
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
|
_is_gfx95_supported = is_gfx95_supported()
|
|
_is_gfx1250_supported = is_gfx1250_supported()
|
|
_is_npu = is_npu()
|
|
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
|
|
|
if _use_aiter:
|
|
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype
|
|
|
|
if _is_gfx1250_supported:
|
|
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
|
else:
|
|
from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant
|
|
from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant
|
|
|
|
if _is_gfx95_supported:
|
|
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant
|
|
|
|
from sglang.srt.layers.quantization.rocm_mxfp4_utils import (
|
|
fused_rms_mxfp4_quant,
|
|
)
|
|
elif _is_npu:
|
|
from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache
|
|
|
|
|
|
def _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
epsilon: float,
|
|
residual: Optional[torch.Tensor] = None,
|
|
):
|
|
"""Fused (optional residual-add +) RMSNorm + FP8 per-token quantization.
|
|
|
|
Only used with the aiter (ROCm) backend.
|
|
|
|
Args:
|
|
residual: if provided, computes hidden_states + residual before RMSNorm
|
|
and returns updated residual_out as second element.
|
|
|
|
Returns:
|
|
If residual is None: (out_fp8, scale)
|
|
If residual provided: ((out_fp8, scale), residual_out)
|
|
"""
|
|
if _is_gfx1250_supported:
|
|
# per-token quant == group quant with group_size == hidden size, giving
|
|
# an (M, 1) scale.
|
|
N = hidden_states.shape[-1]
|
|
(out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant(
|
|
hidden_states,
|
|
weight,
|
|
epsilon,
|
|
group_size=N,
|
|
dtype_quant=_aiter_fp8_dtype,
|
|
res1=residual,
|
|
)
|
|
if residual is not None:
|
|
return (out_fp8, scale), residual_out
|
|
return (out_fp8, scale)
|
|
|
|
M, N = hidden_states.shape
|
|
out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device)
|
|
scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device)
|
|
if residual is not None:
|
|
residual_out = torch.empty_like(hidden_states)
|
|
_aiter_add_rmsnorm_quant(
|
|
out_fp8,
|
|
hidden_states,
|
|
residual,
|
|
residual_out,
|
|
scale,
|
|
weight,
|
|
epsilon,
|
|
0, # group_size=0 → per-token
|
|
)
|
|
return (out_fp8, scale.unsqueeze(1)), residual_out
|
|
else:
|
|
_aiter_rmsnorm_quant(
|
|
out_fp8,
|
|
hidden_states,
|
|
scale,
|
|
weight,
|
|
epsilon,
|
|
0, # group_size=0 → per-token
|
|
)
|
|
return (out_fp8, scale.unsqueeze(1))
|
|
|
|
|
|
# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465
|
|
# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True).
|
|
FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048
|
|
|
|
|
|
def apply_flashinfer_allreduce_fusion(batch_size: int):
|
|
return (
|
|
# NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion
|
|
# Ref: https://github.com/sgl-project/sglang/issues/17237
|
|
(_is_sm90_supported or _is_sm100_supported)
|
|
and _is_flashinfer_available
|
|
and not is_dp_attention_enabled()
|
|
and get_exec().comm.flashinfer_allreduce_fusion_backend is not None
|
|
# cutedsl runs its own fused path from the fusion communicator.
|
|
and not uses_cutedsl_ar_fusion()
|
|
and not is_flashinfer_allreduce_unavailable()
|
|
# Symbolic size checks stay last: under Dynamo tracing they guard on
|
|
# the dynamic token dim, so statically-off configs must short-circuit
|
|
# before reaching them.
|
|
and batch_size > 0
|
|
and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE
|
|
)
|
|
|
|
|
|
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
|
|
comm = get_exec().comm
|
|
if not comm.enable_aiter_allreduce_fusion:
|
|
return False
|
|
if forward_mode.is_extend_or_draft_extend_or_mixed():
|
|
return not comm.disable_aiter_allreduce_fusion_in_prefill
|
|
return not comm.disable_aiter_allreduce_fusion_in_decode
|
|
|
|
|
|
def apply_aiter_all_reduce_fusion(
|
|
input_tensor: torch.Tensor, forward_batch: ForwardBatch
|
|
):
|
|
n = input_tensor.shape[-1]
|
|
total_bytes = input_tensor.numel() * input_tensor.element_size()
|
|
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
|
|
return (
|
|
_use_aiter
|
|
and total_bytes > 0
|
|
and n <= 16384
|
|
and total_bytes <= 8 * 1024 * 8192
|
|
and get_parallel().tp_size != 6
|
|
and not is_dp_attention_enabled()
|
|
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
|
)
|
|
|
|
|
|
class ScatterMode(Enum):
|
|
"""
|
|
Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d
|
|
Model input/output: [ab, ab, cd, cd] for four ranks respectively
|
|
SCATTERED: [a, b, c, d]
|
|
TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group
|
|
FULL: [abcd, abcd, abcd, abcd]
|
|
MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size
|
|
"""
|
|
|
|
SCATTERED = auto()
|
|
TP_ATTN_FULL = auto()
|
|
FULL = auto()
|
|
MOE_FULL = auto()
|
|
|
|
@staticmethod
|
|
def model_input_output():
|
|
"""The scatter mode for model forward pass input and output data"""
|
|
if is_dsa_enable_prefill_cp() or is_mla_cp_enabled():
|
|
return ScatterMode.SCATTERED
|
|
|
|
return ScatterMode.TP_ATTN_FULL
|
|
|
|
|
|
class AttentionInputs:
|
|
def __init__(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
qkv_latent_func: Callable,
|
|
*,
|
|
is_pre_gathered: bool = False,
|
|
):
|
|
self.hidden_states_local = hidden_states
|
|
self.forward_batch = forward_batch
|
|
self.qkv_latent_func = qkv_latent_func
|
|
self.hidden_states_ = None
|
|
self.qkv_latent_ = None
|
|
# When True, hidden_states_local is already attn_tp-gathered upstream
|
|
# (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again.
|
|
self.is_pre_gathered = is_pre_gathered
|
|
|
|
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
|
|
total_tokens = forward_batch.input_ids.shape[0]
|
|
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
|
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
|
return output
|
|
|
|
def fetch_qkv_latent(self):
|
|
if self.qkv_latent_ is not None:
|
|
return self.qkv_latent_
|
|
assert self.qkv_latent_func is not None
|
|
self.qkv_latent_ = self.qkv_latent_func(
|
|
self.hidden_states_local, self.forward_batch
|
|
)
|
|
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
|
self.qkv_latent_ = self.tp_all_gather_hidden_states(
|
|
self.qkv_latent_, self.forward_batch
|
|
)
|
|
return self.qkv_latent_
|
|
|
|
def fetch_hidden_states(self):
|
|
if self.hidden_states_ is not None:
|
|
return self.hidden_states_
|
|
self.hidden_states_ = self.hidden_states_local
|
|
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
|
self.hidden_states_ = self.tp_all_gather_hidden_states(
|
|
self.hidden_states_, self.forward_batch
|
|
)
|
|
return self.hidden_states_
|
|
|
|
|
|
class AttnTpContext:
|
|
def __init__(self):
|
|
self.allow_input_scattered = False
|
|
self.is_dsa = False
|
|
|
|
def init_context(self, q_lora_rank, is_dsa, is_mhc=False):
|
|
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
|
|
# cannot use scattered inputs.
|
|
self.is_dsa = is_dsa
|
|
self.allow_input_scattered = (
|
|
get_parallel().enable_attn_tp_input_scattered
|
|
and (_is_cuda or _is_npu)
|
|
and q_lora_rank is not None
|
|
and (is_mhc or not is_dsa)
|
|
and get_parallel().tp_size > 1
|
|
and not is_dp_attention_enabled()
|
|
and get_moe_a2a_backend().is_none()
|
|
and not enable_moe_dense_fully_dp()
|
|
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
|
and get_spec().speculative_algorithm != "EAGLE3"
|
|
)
|
|
if get_parallel().enable_attn_tp_input_scattered:
|
|
if not self.allow_input_scattered:
|
|
logging.info(
|
|
"attn_tp_input_scattered is not enabled while other conditions are not met"
|
|
)
|
|
else:
|
|
logging.info("attn_tp_input_scattered is enabled")
|
|
|
|
def use_input_scattered(self, forward_batch: ForwardBatch):
|
|
return (
|
|
self.allow_input_scattered
|
|
and forward_batch.forward_mode.is_extend()
|
|
and not forward_batch.forward_mode.is_target_verify()
|
|
and forward_batch.input_ids is not None
|
|
and not forward_batch.can_run_tbo
|
|
)
|
|
|
|
@property
|
|
def input_scattered(self):
|
|
return get_forward().attn_input_scattered
|
|
|
|
def set_attn_inputs(self, attn_inputs: AttentionInputs):
|
|
get_forward().set("attn_inputs", attn_inputs)
|
|
|
|
def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None:
|
|
attn_inputs = get_forward().attn_inputs
|
|
if attn_inputs is not None:
|
|
attn_inputs.hidden_states_local = hidden_states
|
|
|
|
def fetch_qkv_latent(self):
|
|
attn_inputs = get_forward().attn_inputs
|
|
assert attn_inputs is not None
|
|
return attn_inputs.fetch_qkv_latent()
|
|
|
|
def fetch_hidden_states(self):
|
|
attn_inputs = get_forward().attn_inputs
|
|
assert attn_inputs is not None
|
|
return attn_inputs.fetch_hidden_states()
|
|
|
|
def clear_attn_inputs(self) -> None:
|
|
get_forward().set("attn_inputs", None)
|
|
|
|
@contextmanager
|
|
def maybe_input_scattered(self, forward_batch: ForwardBatch):
|
|
flag = self.use_input_scattered(forward_batch)
|
|
forward = get_forward()
|
|
# scoped() also restores when the forward raises — the old in-place
|
|
# swap leaked the flag on exceptions.
|
|
with forward.scoped(attn_input_scattered=flag):
|
|
try:
|
|
yield
|
|
finally:
|
|
forward.set("attn_inputs", None)
|
|
|
|
|
|
ATTN_TP_CONTEXT = AttnTpContext()
|
|
|
|
|
|
def get_attn_tp_context():
|
|
return ATTN_TP_CONTEXT
|
|
|
|
|
|
@dataclass
|
|
class _LayerModeComputationContext:
|
|
num_layers: int
|
|
layer_id: int
|
|
is_layer_sparse: bool
|
|
is_previous_layer_sparse: Optional[bool]
|
|
is_next_layer_sparse: Optional[bool]
|
|
|
|
def previous_layer(self):
|
|
assert self.is_previous_layer_sparse is not None
|
|
return _LayerModeComputationContext(
|
|
num_layers=self.num_layers,
|
|
layer_id=self.layer_id - 1,
|
|
is_layer_sparse=self.is_previous_layer_sparse,
|
|
is_previous_layer_sparse=None,
|
|
is_next_layer_sparse=self.is_layer_sparse,
|
|
)
|
|
|
|
|
|
def sparse_mlp_scatter_mode() -> ScatterMode:
|
|
"""SCATTERED hands a sparse MLP this rank's own token shard; FULL and
|
|
MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group."""
|
|
if (
|
|
# Token dispatch/combine will be handled outside of LayerCommunicator for these modes.
|
|
not get_moe_a2a_backend().is_none()
|
|
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
|
or enable_dwdp()
|
|
):
|
|
return ScatterMode.SCATTERED
|
|
# DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL.
|
|
if is_enable_moe_cp_allgather() and not (
|
|
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
|
):
|
|
return ScatterMode.MOE_FULL
|
|
return ScatterMode.FULL
|
|
|
|
|
|
@dataclass
|
|
class LayerScatterModes:
|
|
layer_input_mode: ScatterMode
|
|
attn_mode: ScatterMode
|
|
# Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed
|
|
mlp_mode: ScatterMode
|
|
middle_residual_mode: ScatterMode
|
|
layer_output_mode: ScatterMode
|
|
is_layer_sparse: bool = False
|
|
# The model's first layer: its input is the embedding, not a layer output.
|
|
is_first_layer: bool = False
|
|
# The model's last layer: its output goes to the final norm, not a next layer.
|
|
is_last_layer: bool = False
|
|
# Whether the layer before this one has a sparse MLP; None when the modes
|
|
# were given directly, not planned from the layer sequence.
|
|
is_previous_layer_sparse: Optional[bool] = None
|
|
|
|
@classmethod
|
|
def init_new(cls, **kwargs):
|
|
context = _LayerModeComputationContext(**kwargs)
|
|
return cls(
|
|
layer_input_mode=cls._compute_layer_input_mode(context),
|
|
attn_mode=ScatterMode.TP_ATTN_FULL,
|
|
mlp_mode=cls._compute_mlp_mode(context),
|
|
middle_residual_mode=cls._compute_middle_residual_mode(context),
|
|
layer_output_mode=cls._compute_layer_output_mode(context),
|
|
is_layer_sparse=context.is_layer_sparse,
|
|
is_first_layer=context.layer_id == 0,
|
|
is_last_layer=context.layer_id == context.num_layers - 1,
|
|
is_previous_layer_sparse=context.is_previous_layer_sparse,
|
|
)
|
|
|
|
@classmethod
|
|
def _compute_layer_input_mode(cls, context: _LayerModeComputationContext):
|
|
if context.layer_id == 0:
|
|
return ScatterMode.model_input_output()
|
|
return cls._compute_layer_output_mode(context.previous_layer())
|
|
|
|
@classmethod
|
|
def _compute_mlp_mode(cls, context: _LayerModeComputationContext):
|
|
if context.is_layer_sparse:
|
|
return sparse_mlp_scatter_mode()
|
|
else:
|
|
if enable_moe_dense_fully_dp():
|
|
return ScatterMode.SCATTERED
|
|
# A TP-sharded dense MLP reduces over the whole TP group, which spans
|
|
# every CP rank; a CP-sharded prefill must gather tokens across CP
|
|
# first or the all-reduce sums different tokens' partial outputs.
|
|
# MLA/DSA CP models do this in DSACPLayerCommunicator instead.
|
|
if _generic_prefill_cp_shards_tokens() and not (
|
|
is_dsa_enable_prefill_cp() or is_mla_cp_enabled()
|
|
):
|
|
return ScatterMode.MOE_FULL
|
|
return ScatterMode.FULL
|
|
|
|
@classmethod
|
|
def _should_gather_for_tbo(cls, context: _LayerModeComputationContext):
|
|
return (
|
|
not context.is_layer_sparse
|
|
and context.is_next_layer_sparse
|
|
and enable_moe_dense_fully_dp()
|
|
and get_exec().overlap.enable_two_batch_overlap
|
|
)
|
|
|
|
@classmethod
|
|
def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext):
|
|
mlp_mode = cls._compute_mlp_mode(context)
|
|
if mlp_mode == ScatterMode.SCATTERED:
|
|
return ScatterMode.SCATTERED
|
|
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def _compute_layer_output_mode(cls, context: _LayerModeComputationContext):
|
|
mlp_mode = cls._compute_mlp_mode(context)
|
|
if context.layer_id == context.num_layers - 1:
|
|
return ScatterMode.model_input_output()
|
|
if mlp_mode == ScatterMode.SCATTERED:
|
|
if cls._should_gather_for_tbo(context):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
return ScatterMode.SCATTERED
|
|
if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL):
|
|
return ScatterMode.TP_ATTN_FULL
|
|
raise NotImplementedError
|
|
|
|
|
|
def enable_moe_dense_fully_dp():
|
|
return get_parallel().moe_dense_tp_size == 1
|
|
|
|
|
|
def _generic_prefill_cp_shards_tokens() -> bool:
|
|
"""Whether the strategy prefill CP path shards prefill tokens across CP ranks."""
|
|
parallel = get_parallel()
|
|
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
|
|
|
|
|
|
def enable_dwdp():
|
|
return get_parallel().dwdp_size > 1
|
|
|
|
|
|
def tp_reduce_scatter(
|
|
hidden_states: torch.Tensor,
|
|
residual: Optional[torch.Tensor],
|
|
context: "CommunicateContext",
|
|
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
|
"""Module-level so MHC communicators can reuse it without a
|
|
``LayerCommunicator`` instance."""
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, hidden_states
|
|
assert hidden_states.shape[0] % context.tp_size == 0, (
|
|
f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0"
|
|
)
|
|
local_tokens = hidden_states.shape[0] // context.tp_size
|
|
output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:])
|
|
get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states)
|
|
if residual is not None:
|
|
residual = residual.tensor_split(context.tp_size)[context.tp_rank]
|
|
return output, residual
|
|
|
|
|
|
def _update_and_read_residual_plain(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
if residual is None:
|
|
return norm(hidden_states), hidden_states
|
|
return norm(hidden_states, residual, post_residual_addition)
|
|
|
|
|
|
def _update_and_read_residual_aiter_mxfp4(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
# post_residual_addition is not applied on this path.
|
|
output, *_, residual_out = fused_rms_mxfp4_quant(
|
|
hidden_states,
|
|
norm.weight,
|
|
norm.variance_epsilon,
|
|
None,
|
|
None,
|
|
None,
|
|
residual,
|
|
)
|
|
return output, hidden_states if residual is None else residual_out
|
|
|
|
|
|
def _update_and_read_residual_aiter_fp8_group(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
"""aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the
|
|
unquantized bf16 output rides along as a third element, so the DSA indexer
|
|
can skip dequantizing. post_residual_addition is not applied on this path."""
|
|
needs_bf16 = get_attn_tp_context().is_dsa
|
|
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
|
|
hidden_states,
|
|
norm.weight,
|
|
norm.variance_epsilon,
|
|
inp2=None,
|
|
inp2_weight=None,
|
|
inp2_epsilon=None,
|
|
group_size=128,
|
|
dtype_quant=torch.float8_e4m3fn,
|
|
res1=residual,
|
|
output_unquantized_inp1=needs_bf16,
|
|
transpose_scale=False,
|
|
)
|
|
if _use_aiter_bpreshuffle_gfx95:
|
|
output = materialize_bpreshuffle_fp8_scale_tuple(output)
|
|
if needs_bf16:
|
|
output = (output[0], output[1], unquantized)
|
|
return output, hidden_states if residual is None else residual_out
|
|
|
|
|
|
def _update_and_read_residual_aiter_fp8_per_token(
|
|
norm, hidden_states, residual, post_residual_addition
|
|
):
|
|
if residual is None:
|
|
output = _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states, norm.weight.data, norm.variance_epsilon
|
|
)
|
|
return output, hidden_states
|
|
if post_residual_addition is not None:
|
|
residual = residual + post_residual_addition
|
|
return _fused_rmsnorm_fp8_per_token_quant(
|
|
hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual
|
|
)
|
|
|
|
|
|
def _attn_input_update_and_read_residual(quant_format: str):
|
|
"""Add the previous layer's output to the residual and read the attention
|
|
input from it: the input norm, fused with the quantization this format
|
|
wants. Without a residual (the first layer, or one already folded into
|
|
hidden_states) the input itself becomes the residual."""
|
|
if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format:
|
|
return _update_and_read_residual_aiter_mxfp4
|
|
if _use_aiter and _is_gfx95_supported and quant_format == "fp8":
|
|
return _update_and_read_residual_aiter_fp8_group
|
|
if _use_aiter and quant_format == "fp8_per_token":
|
|
return _update_and_read_residual_aiter_fp8_per_token
|
|
return _update_and_read_residual_plain
|
|
|
|
|
|
class UnreducedOutput(msgspec.Struct, frozen=True):
|
|
"""A layer output that still owes its sum, left for the next layer's input.
|
|
Hand it to the next layer, or pass it through reduce_output() before reading
|
|
it any other way. The producer says what is owed: one all-reduce over
|
|
``group`` that keeps the layout, or ``reduce_and_redistribute``."""
|
|
|
|
partial: torch.Tensor
|
|
group: Optional[GroupCoordinator] = None
|
|
# Under attention DP: the reduction that also brings ``partial`` back to this
|
|
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
|
|
reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
|
|
|
|
|
def reduce_output(
|
|
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
|
|
) -> Optional[torch.Tensor]:
|
|
"""Run the reduction an UnreducedOutput still owes; pass anything else through."""
|
|
if isinstance(hidden_states, UnreducedOutput):
|
|
if hidden_states.reduce_and_redistribute is not None:
|
|
return hidden_states.reduce_and_redistribute(hidden_states.partial)
|
|
return hidden_states.group.all_reduce(hidden_states.partial)
|
|
return hidden_states
|
|
|
|
|
|
def layer_input_buffer(
|
|
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
|
) -> torch.Tensor:
|
|
"""The tensor holding a layer's input, for reusing its memory without reading it."""
|
|
if isinstance(hidden_states, UnreducedOutput):
|
|
return hidden_states.partial
|
|
return hidden_states
|
|
|
|
|
|
def _batch_size(forward_batch: ForwardBatch) -> int:
|
|
return (
|
|
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
|
|
)
|
|
|
|
|
|
def _deferred_reduction_runs_on_the_tp_group() -> bool:
|
|
"""Whether an MoE output's one all-reduce runs over the TP group object
|
|
itself, the group a dense FFN and reduce_moe_output reduce over."""
|
|
parallel = get_parallel()
|
|
if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1:
|
|
# Some MoE blocks reduce EP and MoE-TP in two steps instead of merging.
|
|
return False
|
|
return post_experts_reduction_group() is parallel.tp_group
|
|
|
|
|
|
def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool:
|
|
if is_dp_attention_enabled():
|
|
# The FFN runs on every DP rank's tokens, so every rank decides alike.
|
|
return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0
|
|
return _batch_size(forward_batch) > 0
|
|
|
|
|
|
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
|
"""Whether the next layer's all-reduce is the one the FFN would have run
|
|
itself: one full-precision sum over the same TP group, on an output
|
|
that is a plain partial sum. The output is not a plain partial sum when the
|
|
MoE combine already summed it, a replicated shared expert is added after the
|
|
reduction, or LoRA-B runs on the unreduced activations."""
|
|
return (
|
|
_ffn_has_tokens(forward_batch)
|
|
and get_moe_a2a_backend().is_none()
|
|
and not post_experts_output_is_complete(is_tp_path=True)
|
|
and not get_exec().comm.enable_quant_communications
|
|
and not envs.SGLANG_SHARED_EXPERT_TP1.get()
|
|
and not get_lora().enable_lora
|
|
and _deferred_reduction_runs_on_the_tp_group()
|
|
)
|
|
|
|
|
|
class LayerCommunicator:
|
|
# Communicators built without __init__ (e.g. test doubles) publish no LoRA layout.
|
|
_publish_lora_layout: bool = False
|
|
# Whether this class's boundary steps may be chosen from both sides'
|
|
# declarations; a subclass that picks its own steps says no.
|
|
_takes_declared_boundaries = True
|
|
|
|
def __init__(
|
|
self,
|
|
layer_scatter_modes: LayerScatterModes,
|
|
input_layernorm: torch.nn.Module,
|
|
post_attention_layernorm: torch.nn.Module,
|
|
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
|
|
allow_reduce_scatter: bool = False,
|
|
qkv_latent_func: Optional[Callable] = None,
|
|
force_layernorm_before_dp_gather: bool = False,
|
|
enable_fused_ar_quant: bool = False,
|
|
fused_ar_quant_keep_bf16: bool = False,
|
|
# False for a layer whose FFN always completes its own reduction.
|
|
allow_deferred_ffn_reduction: bool = True,
|
|
):
|
|
self.layer_scatter_modes = layer_scatter_modes
|
|
self.input_layernorm = input_layernorm
|
|
self.post_attention_layernorm = post_attention_layernorm
|
|
self.allow_reduce_scatter = allow_reduce_scatter
|
|
self.is_last_layer = layer_scatter_modes.is_last_layer
|
|
self.qkv_latent_func = qkv_latent_func
|
|
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
|
|
self.enable_fused_ar_quant = enable_fused_ar_quant
|
|
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
|
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
|
|
|
self._context = CommunicateContext.init_new()
|
|
self._context.force_layernorm_before_dp_gather = (
|
|
force_layernorm_before_dp_gather
|
|
)
|
|
sides = self._declared_sides()
|
|
if sides is not None:
|
|
self._select_declared_boundaries(sides)
|
|
else:
|
|
self._select_boundaries_from_scatter_modes()
|
|
self._attn_input_fusions = self._select_attn_input_fusions()
|
|
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
|
get_spec().speculative_algorithm
|
|
)
|
|
# LoRA kernels need the per-layer token layout only under DP attention.
|
|
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
|
get_lora().enable_lora
|
|
)
|
|
|
|
# Under LayerNorm SP the residual and norms run on this rank's sequence
|
|
# shard with no collectives while the region is active.
|
|
self._sp_region = layernorm_sp.layernorm_sp_enabled()
|
|
|
|
def _declared_sides(self) -> Optional[DecoderLayerSides]:
|
|
"""The declarations this layer's boundaries are chosen from: an
|
|
attention and a dense MLP on the TP group under attention DP, without CP
|
|
or LayerNorm SP, in a layer that takes the rows a dense layer or the
|
|
embedding hands on. None when the steps come from the scatter modes."""
|
|
modes = self.layer_scatter_modes
|
|
parallel = get_parallel()
|
|
if not (
|
|
self._takes_declared_boundaries
|
|
and parallel.attn_dp_size > 1
|
|
and parallel.attn_cp_size == 1
|
|
and not layernorm_sp.layernorm_sp_enabled()
|
|
and not modes.is_layer_sparse
|
|
and not enable_moe_dense_fully_dp()
|
|
and (modes.is_first_layer or modes.is_previous_layer_sparse is False)
|
|
):
|
|
return None
|
|
return dense_decoder_layer_sides(
|
|
axis_sizes={
|
|
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
|
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
|
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
|
},
|
|
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
|
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
|
)
|
|
|
|
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
|
|
"""Choose the three boundary sides this layer owns from the
|
|
declarations: the attention input, the attention -> FFN steps, and the
|
|
FFN output's way back to the layer's rows."""
|
|
self._communicate_simple_fn = _select_attention_input_move(
|
|
sides.layer_rows, sides.attention
|
|
)
|
|
self._mlp_input = _select_ffn_input(
|
|
sides.attention_output,
|
|
residual=sides.layer_rows,
|
|
need=sides.ffn,
|
|
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
|
)
|
|
# Neither fused kernel runs under attention DP.
|
|
self._mlp_input_may_fuse = False
|
|
self._ffn_output = sides.ffn_output
|
|
self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp(
|
|
sides.ffn_output, sides.layer_rows
|
|
)
|
|
self._ffn_sum_is_movable = sides.ffn_output.group is not None
|
|
|
|
def _select_boundaries_from_scatter_modes(self) -> None:
|
|
self._post_init_communicate()
|
|
# Under attention DP, the base postprocess scatters the FFN output back
|
|
# to this rank's tokens, which the next layer's input can run instead;
|
|
# the MHC and DSA-CP postprocess do more, so theirs stays here.
|
|
self._postprocess_scatters_to_local_tokens = (
|
|
self._communicate_summable_tensor_pair_fn
|
|
is CommunicateSummableTensorPairFn._scatter_hidden_states
|
|
)
|
|
modes = self.layer_scatter_modes
|
|
self._ffn_output = StageOutput(
|
|
self._context.layouts[modes.mlp_mode],
|
|
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
|
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
|
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
|
# A MoE block leaves its sum to reduce_scatterv whenever it applies
|
|
# (should_skip_post_experts_all_reduce); a dense MLP does only under
|
|
# the published mlp_reduce_scatter.
|
|
leaves_for_reduce_scatterv=(
|
|
self.allow_reduce_scatter or modes.is_layer_sparse
|
|
),
|
|
)
|
|
# Whether the FFN sum is one the next layer's input can take: not when
|
|
# the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes
|
|
# whole tokens with nothing left to sum.
|
|
self._ffn_sum_is_movable = modes.mlp_mode not in (
|
|
ScatterMode.MOE_FULL,
|
|
ScatterMode.SCATTERED,
|
|
)
|
|
self._mlp_input, self._mlp_input_may_fuse = self._select_mlp_input()
|
|
|
|
def _post_init_communicate(self):
|
|
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
|
input_mode=self.layer_scatter_modes.layer_input_mode,
|
|
output_mode=self.layer_scatter_modes.attn_mode,
|
|
context=self._context,
|
|
)
|
|
self._communicate_summable_tensor_pair_fn = (
|
|
CommunicateSummableTensorPairFn.get_fn(
|
|
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
|
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
|
output_mode=self.layer_scatter_modes.layer_output_mode,
|
|
context=self._context,
|
|
)
|
|
)
|
|
|
|
def prepare_attn_and_capture_last_layer_outputs(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
|
post_residual_addition: Optional[torch.Tensor] = None,
|
|
quant_format: str = "",
|
|
):
|
|
hidden_states, residual = self.prepare_attn(
|
|
hidden_states,
|
|
residual,
|
|
forward_batch,
|
|
quant_format=quant_format,
|
|
post_residual_addition=post_residual_addition,
|
|
)
|
|
if captured_last_layer_outputs is not None:
|
|
gathered_last_layer_output = self._communicate_simple_fn(
|
|
hidden_states=residual,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
)
|
|
if (
|
|
gathered_last_layer_output is residual
|
|
# An accumulator that copies on append already holds a snapshot.
|
|
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
|
|
and not self._post_attn_residual_is_read_only(residual)
|
|
):
|
|
gathered_last_layer_output = residual.clone()
|
|
captured_last_layer_outputs.append(gathered_last_layer_output)
|
|
return hidden_states, residual
|
|
|
|
def _post_attn_residual_is_read_only(self, residual: torch.Tensor) -> bool:
|
|
"""True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual``
|
|
untouched, so Eagle3 aux capture can keep its reference and skip the clone.
|
|
|
|
Only the flashinfer all-reduce-fusion kernel writes a fresh
|
|
``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter
|
|
fused kernel and every plain norm fold into ``residual`` in place. It runs
|
|
when ``prepare_mlp`` selected its fused entry and the batch is not
|
|
input-scattered.
|
|
"""
|
|
return (
|
|
self._mlp_input_may_fuse
|
|
and not get_attn_tp_context().input_scattered
|
|
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
|
)
|
|
|
|
def publish_attn_lora_layout(self) -> None:
|
|
"""Attention consumes the DP-local token batch."""
|
|
if self._publish_lora_layout:
|
|
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
|
|
|
|
def publish_mlp_lora_layout(self) -> None:
|
|
"""The MLP consumes the TP-global batch only after a FULL DP gather."""
|
|
if self._publish_lora_layout:
|
|
get_forward().set(
|
|
"lora_batch_layout",
|
|
(
|
|
LoRABatchLayout.TP_GLOBAL
|
|
if self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
|
|
else LoRABatchLayout.DP_LOCAL
|
|
),
|
|
)
|
|
|
|
def prepare_attn(
|
|
self,
|
|
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
quant_format: str = "",
|
|
post_residual_addition: Optional[torch.Tensor] = None,
|
|
):
|
|
self.publish_attn_lora_layout()
|
|
|
|
# What the previous layer left to complete: an UnreducedOutput, or a
|
|
# producer's handoff that one of the fused entries consumes.
|
|
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
|
|
if owed is not None and residual is None:
|
|
raise RuntimeError(f"{type(owed).__name__} requires residual input")
|
|
if (
|
|
isinstance(owed, UnreducedOutput)
|
|
and owed.reduce_and_redistribute is not None
|
|
):
|
|
# No fused kernel runs under attention DP: the reduce-scatter back to
|
|
# this rank's tokens comes first.
|
|
hidden_states, owed = reduce_output(owed), None
|
|
if owed is not None:
|
|
for fused in self._attn_input_fusions:
|
|
result = fused(owed, residual, forward_batch, post_residual_addition)
|
|
if result is not None:
|
|
return self._finish_prepare_attn(
|
|
hidden_states=result[0],
|
|
residual=result[1],
|
|
forward_batch=forward_batch,
|
|
)
|
|
hidden_states = owed.partial
|
|
# The SP region opens at the first layer, re-evaluated per forward so a
|
|
# crash mid-loop cannot leak into the next one.
|
|
if self._sp_region and self.layer_scatter_modes.is_first_layer:
|
|
get_forward().set(
|
|
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
|
)
|
|
if get_forward().sp_active:
|
|
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
|
if get_attn_tp_context().input_scattered:
|
|
hidden_states, residual = self._tp_reduce_scatter(
|
|
hidden_states,
|
|
residual,
|
|
)
|
|
if hidden_states.shape[0] == 0:
|
|
residual = hidden_states
|
|
elif owed is not None:
|
|
hidden_states = reduce_output(owed)
|
|
hidden_states, residual = _attn_input_update_and_read_residual(
|
|
quant_format
|
|
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
|
else:
|
|
hidden_states, residual = _attn_input_update_and_read_residual(
|
|
quant_format
|
|
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
|
|
|
return self._finish_prepare_attn(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
)
|
|
|
|
def _in_sp_region(self) -> bool:
|
|
return self._sp_region and get_forward().sp_active
|
|
|
|
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
|
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
|
|
the SP region the attention input stays on this rank's sequence shard."""
|
|
if not self._in_sp_region():
|
|
hidden_states = self._communicate_simple_fn(
|
|
hidden_states=hidden_states,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
)
|
|
if self.qkv_latent_func is not None:
|
|
attn_inputs = AttentionInputs(
|
|
hidden_states, forward_batch, self.qkv_latent_func
|
|
)
|
|
get_attn_tp_context().set_attn_inputs(attn_inputs)
|
|
return hidden_states, residual
|
|
|
|
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
|
|
"""The fused kernels that complete what the previous layer left together
|
|
with the residual update and the input norm, in the order they are tried.
|
|
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
|
returns None when it does not take the batch."""
|
|
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
|
return ()
|
|
self._attn_input_fuses_quant = (
|
|
self.enable_fused_ar_quant
|
|
and _use_aiter
|
|
and hasattr(
|
|
self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group"
|
|
)
|
|
)
|
|
return (self._reduce_output_and_update_and_read_residual,)
|
|
|
|
def _reduce_output_and_update_and_read_residual(
|
|
self,
|
|
owed: UnreducedOutput,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
post_residual_addition: Optional[torch.Tensor],
|
|
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
|
"""Complete the sum the previous layer left, add it to the residual and
|
|
apply the input norm in one aiter or flashinfer kernel; None when the
|
|
kernel does not take this batch. The result is not quantized for
|
|
``quant_format``."""
|
|
if (
|
|
not isinstance(owed, UnreducedOutput)
|
|
# The kernel does not add it.
|
|
or post_residual_addition is not None
|
|
# The kernel reduces over the MoE output's group.
|
|
or owed.group is not post_experts_reduction_group()
|
|
):
|
|
return None
|
|
hidden_states = owed.partial
|
|
if not (
|
|
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
|
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
|
):
|
|
return None
|
|
if self._attn_input_fuses_quant:
|
|
# Falls back to AR+RMSNorm + separate quant internally when the
|
|
# fully-fused kernel cannot service the shape.
|
|
quant_result = (
|
|
self.input_layernorm.forward_with_allreduce_fusion_quant_per_group(
|
|
hidden_states,
|
|
residual,
|
|
use_attn_tp_group=False,
|
|
keep_bf16=self.fused_ar_quant_keep_bf16,
|
|
)
|
|
)
|
|
if quant_result is not None:
|
|
return quant_result
|
|
return self.input_layernorm.forward_with_allreduce_fusion(
|
|
hidden_states, residual, use_attn_tp_group=False
|
|
)
|
|
|
|
def _tp_reduce_scatter(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
return tp_reduce_scatter(hidden_states, residual, self._context)
|
|
|
|
def _select_mlp_input(self) -> Tuple[Callable, bool]:
|
|
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
|
whether they include the aiter / flashinfer fused entry. The MHC and
|
|
DSA-CP communicators pick their own implementations."""
|
|
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
|
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
|
if kind is MlpInputKind.NORM:
|
|
return _mlp_input_norm, False
|
|
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
|
return _mlp_input_attn_tp_all_reduce, False
|
|
if kind is MlpInputKind.SCATTER:
|
|
return (
|
|
partial(
|
|
_mlp_input_scatter,
|
|
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
|
),
|
|
False,
|
|
)
|
|
# Neither fused kernel runs under attention DP.
|
|
fusions = (
|
|
self._select_mlp_input_fusions(residual_input_mode)
|
|
if self._context.attn_dp_size == 1
|
|
else ()
|
|
)
|
|
steps = partial(
|
|
_mlp_input_gather,
|
|
order=_mlp_input_order(self._context, residual_input_mode, fusions),
|
|
)
|
|
if kind is MlpInputKind.GATHER_MOE_CP:
|
|
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
|
|
return (
|
|
steps,
|
|
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
|
|
)
|
|
|
|
def _select_mlp_input_fusions(
|
|
self, residual_input_mode: ScatterMode
|
|
) -> Tuple[Callable, ...]:
|
|
"""The fused kernels that complete the attention output's all-reduce
|
|
together with the residual update and the post-attention norm, in the
|
|
order they are tried. Each takes (hidden_states, residual, forward_batch)
|
|
and returns None when it does not take the batch."""
|
|
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
|
return ()
|
|
return (self._mlp_input_reduce_output_and_update_and_read_residual,)
|
|
|
|
def _mlp_input_reduce_output_and_update_and_read_residual(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
|
"""The attention-TP all-reduce, residual add and norm in one aiter or
|
|
flashinfer kernel; None when neither takes the batch."""
|
|
if not (
|
|
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
|
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
|
):
|
|
return None
|
|
return self.post_attention_layernorm.forward_with_allreduce_fusion(
|
|
hidden_states, residual, use_attn_tp_group=True
|
|
)
|
|
|
|
def prepare_mlp(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
cache=None,
|
|
):
|
|
self.publish_mlp_lora_layout()
|
|
if self._in_sp_region():
|
|
return _mlp_input_norm(
|
|
hidden_states,
|
|
residual,
|
|
forward_batch,
|
|
self.post_attention_layernorm,
|
|
self._context,
|
|
)
|
|
if cache is not None:
|
|
self._context.cache = cache
|
|
|
|
return self._mlp_input(
|
|
hidden_states,
|
|
residual,
|
|
forward_batch,
|
|
self.post_attention_layernorm,
|
|
self._context,
|
|
)
|
|
|
|
def maybe_prefetch_next_full_attention_kv(
|
|
self,
|
|
forward_batch: ForwardBatch,
|
|
next_full_attention_layer_id: Optional[int],
|
|
) -> None:
|
|
return
|
|
|
|
def postprocess_layer(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
):
|
|
if self._in_sp_region():
|
|
return hidden_states, residual
|
|
if self._postprocess_scatters_to_local_tokens:
|
|
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
|
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
|
return self._communicate_summable_tensor_pair_fn(
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
forward_batch=forward_batch,
|
|
context=self._context,
|
|
allow_reduce_scatter=self.allow_reduce_scatter,
|
|
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
|
)
|
|
|
|
def _local_token_move_can_go_to_next_layer(self) -> bool:
|
|
"""Whether the next layer's input can run this layer's move of its FFN
|
|
output back to this rank's tokens: the base postprocess scatter, outside
|
|
an active LayerNorm SP region."""
|
|
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
|
|
|
|
def _postprocess_dp_step(
|
|
self, forward_batch: ForwardBatch
|
|
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
|
"""The reduce-scatter that brings this layer's FFN output back to this
|
|
rank's tokens under attention DP; None when the base postprocess would
|
|
only scatter, or does not move tokens."""
|
|
if not self._postprocess_scatters_to_local_tokens:
|
|
return None
|
|
return _reduce_and_redistribute_output_step(
|
|
forward_batch,
|
|
leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter,
|
|
leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv,
|
|
)
|
|
|
|
def _ffn_leaves_sum_to_reduce_scatter(
|
|
self, forward_batch: ForwardBatch, dp_step: Optional[Callable]
|
|
) -> bool:
|
|
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
|
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
|
one."""
|
|
if not self._ffn_output.leaves_for_reduce_scatter:
|
|
return False
|
|
if dp_step is not None:
|
|
return True
|
|
# Prefill CP predicates must stay out of decode graph capture.
|
|
if forward_batch.forward_mode.is_context_parallel_extend() and (
|
|
dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch)
|
|
):
|
|
return True
|
|
return get_attn_tp_context().input_scattered and not self.is_last_layer
|
|
|
|
def ffn_reduction_group(self) -> GroupCoordinator:
|
|
"""The group this layer's FFN output owes its sum over: the MoE output's
|
|
group on a sparse layer, the TP group a dense MLP reduces over."""
|
|
return _sum_group(self._ffn_output.group)
|
|
|
|
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
|
|
"""Decide once, before the FFN runs, what it skips and what completes its
|
|
output: the next layer's input, or this layer's postprocess step."""
|
|
dp_step = self._postprocess_dp_step(forward_batch)
|
|
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
|
|
forward_batch, dp_step
|
|
)
|
|
complete_now = partial(
|
|
self._complete_ffn_output_now,
|
|
forward_batch=forward_batch,
|
|
dp_step=dp_step,
|
|
)
|
|
if not self._ffn_output.leaves_for_next_layer:
|
|
return FfnCompletion(
|
|
defer_moe_finalize=False,
|
|
fuse_mlp_allreduce=False,
|
|
mlp_reduce_scatter=mlp_reduce_scatter,
|
|
complete=complete_now,
|
|
)
|
|
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
|
|
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
|
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
|
forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step
|
|
)
|
|
if fuse_mlp_allreduce:
|
|
group = self.ffn_reduction_group()
|
|
if self._postprocess_scatters_to_local_tokens:
|
|
# Under attention DP the next layer also brings the sum back to
|
|
# this rank's tokens.
|
|
wrap = partial(
|
|
UnreducedOutput,
|
|
reduce_and_redistribute=partial(
|
|
_all_reduce_then_to_local_tokens, group, forward_batch
|
|
),
|
|
)
|
|
else:
|
|
wrap = partial(UnreducedOutput, group=group)
|
|
complete = partial(_leave_to_next_layer, wrap)
|
|
elif (
|
|
dp_step is not None
|
|
and not self.is_last_layer
|
|
and self._local_token_move_can_go_to_next_layer()
|
|
):
|
|
complete = partial(
|
|
_leave_to_next_layer,
|
|
partial(
|
|
UnreducedOutput,
|
|
reduce_and_redistribute=partial(
|
|
_to_local_tokens, dp_step, forward_batch
|
|
),
|
|
),
|
|
)
|
|
else:
|
|
complete = complete_now
|
|
return FfnCompletion(
|
|
defer_moe_finalize=defer_moe_finalize,
|
|
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
|
mlp_reduce_scatter=mlp_reduce_scatter,
|
|
complete=complete,
|
|
)
|
|
|
|
def _complete_ffn_output_now(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
*,
|
|
forward_batch: ForwardBatch,
|
|
dp_step: Optional[Callable],
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""This layer's postprocess, run with the attention-DP step the FFN exit
|
|
chose."""
|
|
if self._local_token_move_can_go_to_next_layer():
|
|
return (
|
|
_to_local_tokens(
|
|
dp_step or _redistribute_output, forward_batch, hidden_states
|
|
),
|
|
residual,
|
|
)
|
|
return self.postprocess_layer(hidden_states, residual, forward_batch)
|
|
|
|
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
|
"""Decide once how this layer's FFN output reduction completes. Use the
|
|
result as a context manager around the FFN call, then call ``finish``."""
|
|
return FfnExit(self, forward_batch)
|
|
|
|
def finish_layer_stack(
|
|
self,
|
|
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
|
residual: Optional[torch.Tensor],
|
|
forward_batch: ForwardBatch,
|
|
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
|
"""Complete what this layer left for a next layer. Call it on the last
|
|
layer of this rank before its output reaches the final norm, the next
|
|
pipeline rank, or any other consumer outside the layers."""
|
|
return reduce_output(hidden_states), residual
|
|
|
|
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
|
"""Whether the FFN leaves its sum to a reduce-scatter, for layers that run
|
|
their FFN outside ffn_exit."""
|
|
return self._ffn_leaves_sum_to_reduce_scatter(
|
|
forward_batch, self._postprocess_dp_step(forward_batch)
|
|
)
|
|
|
|
def should_defer_moe_finalize(
|
|
self, forward_batch: ForwardBatch, m: int | None = None
|
|
) -> bool:
|
|
"""Whether the MoE may hand an unfinalized output to the next layer."""
|
|
return False
|
|
|
|
def _ffn_sum_can_move_to_next_layer(self) -> bool:
|
|
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
|
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
|
if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable:
|
|
return False
|
|
|
|
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
|
# two disjoint groups; post_experts_all_reduce() merges them into one
|
|
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
|
|
# When merging is blocked, no single group covers both, so fusion stays off.
|
|
parallel = get_parallel()
|
|
if (
|
|
parallel.moe_ep_size > 1
|
|
and parallel.moe_tp_size > 1
|
|
and not can_merge_post_experts_all_reduce()
|
|
):
|
|
return False
|
|
|
|
if (
|
|
is_dp_attention_enabled()
|
|
and self._speculative_algo is not None
|
|
and self._speculative_algo.is_eagle()
|
|
):
|
|
return False
|
|
|
|
return not get_attn_tp_context().input_scattered
|
|
|
|
# NOTE: This function will cause torch recompilation
|
|
def should_fuse_mlp_allreduce_with_next_layer(
|
|
self, forward_batch: ForwardBatch
|
|
) -> bool:
|
|
if not self._ffn_sum_can_move_to_next_layer():
|
|
return False
|
|
batch_size = _batch_size(forward_batch)
|
|
return (
|
|
(
|
|
apply_flashinfer_allreduce_fusion(batch_size)
|
|
or (
|
|
_use_aiter
|
|
and batch_size > 0
|
|
and get_parallel().tp_size != 6
|
|
and not is_dp_attention_enabled()
|
|
and get_moe_a2a_backend().is_none()
|
|
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
|
)
|
|
)
|
|
and (not self.is_last_layer)
|
|
and (self._context.tp_size > 1)
|
|
)
|
|
|
|
def _ffn_sum_moves_to_next_layer(
|
|
self,
|
|
forward_batch: ForwardBatch,
|
|
*,
|
|
mlp_reduce_scatter: bool,
|
|
dp_step: Optional[Callable],
|
|
) -> bool:
|
|
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
|
input norm: whenever the fused kernel takes it, and otherwise when the
|
|
next layer would run the same all-reduce the FFN itself would have."""
|
|
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
|
|
return True
|
|
return (
|
|
self._context.tp_size > 1
|
|
and not self.is_last_layer
|
|
and self._ffn_sum_can_move_to_next_layer()
|
|
and _unfused_completion_matches_the_ffn(forward_batch)
|
|
and not mlp_reduce_scatter
|
|
# Under attention DP the next layer must also run postprocess's
|
|
# scatter back to this rank's tokens, and nothing more.
|
|
and (
|
|
not is_dp_attention_enabled()
|
|
or (self._local_token_move_can_go_to_next_layer() and dp_step is None)
|
|
)
|
|
)
|
|
|
|
|
|
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
|
|
_SCATTER_MODE_SHARDED_AXES = {
|
|
ScatterMode.SCATTERED: (
|
|
TokenAxis.ATTN_DP,
|
|
TokenAxis.ATTN_CP,
|
|
TokenAxis.ATTN_TP_SCATTER,
|
|
),
|
|
ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP),
|
|
ScatterMode.FULL: (TokenAxis.ATTN_CP,),
|
|
ScatterMode.MOE_FULL: (),
|
|
}
|
|
|
|
|
|
def scatter_mode_layouts(
|
|
*, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int
|
|
) -> Dict[ScatterMode, Layout]:
|
|
axis_sizes = {
|
|
TokenAxis.ATTN_DP: attn_dp_size,
|
|
TokenAxis.ATTN_CP: attn_cp_size,
|
|
TokenAxis.ATTN_TP_SCATTER: attn_tp_size,
|
|
}
|
|
return {
|
|
mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes)
|
|
for mode, axes in _SCATTER_MODE_SHARDED_AXES.items()
|
|
}
|
|
|
|
|
|
class FfnCompletion(msgspec.Struct, frozen=True):
|
|
"""One FFN's reduction decision. The flags are published while the FFN runs;
|
|
``complete(hidden_states, residual)`` then either wraps the output for the
|
|
next layer's input to complete, or runs this layer's postprocess step."""
|
|
|
|
defer_moe_finalize: bool
|
|
fuse_mlp_allreduce: bool
|
|
mlp_reduce_scatter: bool
|
|
complete: Callable[[torch.Tensor, torch.Tensor], Tuple]
|
|
|
|
|
|
def _leave_to_next_layer(
|
|
wrap: Callable[[torch.Tensor], UnreducedOutput],
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
) -> Tuple[UnreducedOutput, torch.Tensor]:
|
|
return wrap(hidden_states), residual
|
|
|
|
|
|
class FfnExit:
|
|
"""The scope that publishes an FfnCompletion while the FFN runs: inside the
|
|
``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` /
|
|
``defer_moe_finalize`` on ``get_forward()``."""
|
|
|
|
__slots__ = (
|
|
"communicator",
|
|
"forward_batch",
|
|
"defer_moe_finalize",
|
|
"fuse_mlp_allreduce",
|
|
"mlp_reduce_scatter",
|
|
"_complete",
|
|
"_scope",
|
|
)
|
|
|
|
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
|
self.communicator = communicator
|
|
self.forward_batch = forward_batch
|
|
completion = communicator._select_ffn_completion(forward_batch)
|
|
self.defer_moe_finalize = completion.defer_moe_finalize
|
|
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
|
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
|
self._complete = completion.complete
|
|
self._scope = get_forward().scoped(
|
|
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
|
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
|
defer_moe_finalize=self.defer_moe_finalize,
|
|
)
|
|
|
|
def __enter__(self) -> "FfnExit":
|
|
self._scope.__enter__()
|
|
return self
|
|
|
|
def __exit__(self, *exc_info):
|
|
return self._scope.__exit__(*exc_info)
|
|
|
|
def finish(
|
|
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
|
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
|
"""Leave the reduction to the next layer's input, or postprocess."""
|
|
if not isinstance(hidden_states, torch.Tensor):
|
|
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
|
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
|
return hidden_states, residual
|
|
return self._complete(hidden_states, residual)
|
|
|
|
|
|
@dataclass
|
|
class CommunicateContext:
|
|
process_group_sizes: Dict[ScatterMode, int]
|
|
attn_tp_rank: int
|
|
attn_tp_size: int
|
|
attn_dp_size: int
|
|
attn_cp_rank: int
|
|
attn_cp_size: int
|
|
tp_size: int
|
|
cache = None
|
|
tp_rank: int
|
|
force_layernorm_before_dp_gather: bool = False
|
|
|
|
def is_same_group_size(self, a: ScatterMode, b: ScatterMode):
|
|
return self.process_group_sizes[a] == self.process_group_sizes[b]
|
|
|
|
@cached_property
|
|
def layouts(self) -> Dict[ScatterMode, Layout]:
|
|
return scatter_mode_layouts(
|
|
attn_dp_size=self.attn_dp_size,
|
|
attn_cp_size=self.attn_cp_size,
|
|
attn_tp_size=self.attn_tp_size,
|
|
)
|
|
|
|
def is_same_layout(self, a: ScatterMode, b: ScatterMode):
|
|
return self.layouts[a] == self.layouts[b]
|
|
|
|
@classmethod
|
|
def init_new(cls):
|
|
attn_tp_rank = get_parallel().attn_tp_rank
|
|
attn_tp_size = get_parallel().attn_tp_size
|
|
attn_dp_size = get_parallel().attn_dp_size
|
|
attn_cp_size = get_parallel().attn_cp_size
|
|
attn_cp_rank = get_parallel().attn_cp_rank
|
|
tp_size = get_parallel().tp_size
|
|
tp_rank = get_parallel().tp_rank
|
|
moe_cp_size = get_moe_cp_size()
|
|
process_group_sizes = {
|
|
ScatterMode.SCATTERED: 1,
|
|
ScatterMode.TP_ATTN_FULL: attn_tp_size,
|
|
# TODO: support --moe-dense-tp-size > 1
|
|
# With context parallel enabled, we should exclude
|
|
# the attn_cp_size from the total tp_size
|
|
ScatterMode.FULL: tp_size // attn_cp_size,
|
|
ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size),
|
|
}
|
|
return cls(
|
|
process_group_sizes=process_group_sizes,
|
|
attn_tp_rank=attn_tp_rank,
|
|
attn_tp_size=attn_tp_size,
|
|
attn_dp_size=attn_dp_size,
|
|
attn_cp_rank=attn_cp_rank,
|
|
attn_cp_size=attn_cp_size,
|
|
tp_size=tp_size,
|
|
tp_rank=tp_rank,
|
|
)
|
|
|
|
|
|
class CommunicateSimpleFn:
|
|
@staticmethod
|
|
def get_fn(
|
|
input_mode: ScatterMode,
|
|
output_mode: ScatterMode,
|
|
context: CommunicateContext,
|
|
):
|
|
if context.is_same_layout(input_mode, output_mode):
|
|
return CommunicateSimpleFn._trivial
|
|
|
|
if (input_mode == ScatterMode.SCATTERED) and (
|
|
output_mode == ScatterMode.TP_ATTN_FULL
|
|
):
|
|
if _use_ag_after_qlora:
|
|
return CommunicateSimpleFn._trivial
|
|
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
|
|
|
raise NotImplementedError(f"{input_mode=} {output_mode=}")
|
|
|
|
@staticmethod
|
|
def _trivial(
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
) -> torch.Tensor:
|
|
return hidden_states
|
|
|
|
@staticmethod
|
|
def _scattered_to_tp_attn_full(
|
|
hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
|
if isinstance(hidden_states, tuple):
|
|
gathered_hidden_states = []
|
|
for local_hidden_states in hidden_states:
|
|
with use_symmetric_memory(
|
|
get_parallel().tp_group,
|
|
disabled=not is_allocation_symmetric(),
|
|
):
|
|
output = torch.empty(
|
|
(
|
|
local_hidden_states.shape[0] * context.attn_tp_size,
|
|
*local_hidden_states.shape[1:],
|
|
),
|
|
dtype=local_hidden_states.dtype,
|
|
device=local_hidden_states.device,
|
|
)
|
|
attn_tp_all_gather_into_tensor(
|
|
output,
|
|
local_hidden_states,
|
|
)
|
|
gathered_hidden_states.append(output)
|
|
return tuple(gathered_hidden_states)
|
|
|
|
return _redistribute_from_attn_tp_shards(hidden_states)
|
|
|
|
|
|
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
|
gathered = get_local_dp_buffer(get_parallel().attn_tp_group)
|
|
attn_tp_all_gather_into_tensor(gathered, tensor)
|
|
return gathered
|
|
|
|
|
|
def _redistribute_to_attn_tp_shards(
|
|
tensor: torch.Tensor, context: CommunicateContext
|
|
) -> torch.Tensor:
|
|
return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
|
|
|
|
|
def _reduce_and_redistribute_output_to_attn_tp_shards(
|
|
hidden_states: torch.Tensor, context: CommunicateContext
|
|
) -> torch.Tensor:
|
|
local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
|
context.attn_tp_rank
|
|
]
|
|
attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states)
|
|
return local_hidden_states
|
|
|
|
|
|
def _redistribute_input_to_moe_cp(
|
|
hidden_states: torch.Tensor, forward_batch: ForwardBatch, moe_cp_size: int
|
|
) -> torch.Tensor:
|
|
# Zigzag split can produce unequal token counts across CP ranks
|
|
# (when seq_len % (cp_size * 2) != 0). NCCL allgather requires
|
|
# equal input sizes, so pad to the max per-rank token count.
|
|
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
|
|
max_tokens = max(per_rank_tokens)
|
|
pad_size = max_tokens - hidden_states.shape[0]
|
|
if pad_size > 0:
|
|
hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size])
|
|
|
|
output = torch.empty(
|
|
(max_tokens * moe_cp_size, hidden_states.shape[1]),
|
|
dtype=hidden_states.dtype,
|
|
device=hidden_states.device,
|
|
)
|
|
moe_cp_all_gather_into_tensor(output, hidden_states)
|
|
return output
|
|
|
|
|
|
def _mlp_input_reduce_output(
|
|
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
|
) -> torch.Tensor:
|
|
if (
|
|
not forward_batch.forward_mode.is_decode_or_idle()
|
|
and get_exec().comm.enable_quant_communications
|
|
):
|
|
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
|
|
return attention_tensor_model_parallel_all_reduce(hidden_states)
|
|
|
|
|
|
def _redistribute_input_to_dp(
|
|
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
|
) -> torch.Tensor:
|
|
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
|
|
dp_gather_replicate(global_hidden_states, hidden_states, forward_batch)
|
|
return global_hidden_states
|
|
|
|
|
|
def _reduce_and_redistribute_output_to_dp(
|
|
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
|
) -> torch.Tensor:
|
|
global_hidden_states = get_global_dp_buffer(get_parallel().tp_group)
|
|
dp_gather_partial(global_hidden_states, hidden_states, forward_batch)
|
|
return global_hidden_states
|
|
|
|
|
|
def _mlp_input_without_dp(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
gathers_residual: bool,
|
|
fusions: Tuple[Callable, ...],
|
|
):
|
|
if gathers_residual:
|
|
residual = _redistribute_from_attn_tp_shards(residual)
|
|
for fused in fusions:
|
|
result = fused(hidden_states, residual, forward_batch)
|
|
if result is not None:
|
|
return result
|
|
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
|
|
if _is_npu and context.cache is not None:
|
|
_ = prepare_weight_cache(hidden_states, context.cache)
|
|
return layernorm(hidden_states, residual)
|
|
|
|
|
|
def _mlp_input_dp_replicate(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
gathers_residual: bool,
|
|
reduces_attention_tp: bool,
|
|
):
|
|
"""Attention DP: complete the attention-TP sum if it is owed, add and
|
|
normalize locally, then gather."""
|
|
if gathers_residual:
|
|
residual = _redistribute_from_attn_tp_shards(residual)
|
|
if hidden_states.shape[0] != 0:
|
|
if reduces_attention_tp:
|
|
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
|
with use_symmetric_memory(
|
|
get_parallel().tp_group,
|
|
disabled=not is_allocation_symmetric(),
|
|
):
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return _redistribute_input_to_dp(hidden_states, forward_batch), residual
|
|
|
|
|
|
def _mlp_input_dp_partial(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
gathers_residual: bool,
|
|
):
|
|
"""Attention DP: one rank adds the residual, the gather sums it, then
|
|
normalize."""
|
|
if gathers_residual:
|
|
residual = _redistribute_from_attn_tp_shards(residual)
|
|
if context.attn_tp_rank == 0:
|
|
hidden_states += residual
|
|
hidden_states = _reduce_and_redistribute_output_to_dp(hidden_states, forward_batch)
|
|
dp_scatter(residual, hidden_states, forward_batch)
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states = layernorm(hidden_states)
|
|
return hidden_states, residual
|
|
|
|
|
|
def _mlp_input_order(
|
|
context: CommunicateContext,
|
|
residual_input_mode: ScatterMode,
|
|
fusions: Tuple[Callable, ...],
|
|
) -> Callable:
|
|
"""The steps from the attention output to the FFN input, chosen from facts
|
|
fixed at construction."""
|
|
gathers_residual = (
|
|
residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1
|
|
)
|
|
if context.attn_dp_size == 1:
|
|
return partial(
|
|
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
|
|
)
|
|
if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1:
|
|
return partial(
|
|
_mlp_input_dp_replicate,
|
|
gathers_residual=gathers_residual,
|
|
reduces_attention_tp=context.attn_tp_size > 1,
|
|
)
|
|
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
|
|
|
|
|
def _sum_group(group: SumGroup) -> GroupCoordinator:
|
|
parallel = get_parallel()
|
|
if group is SumGroup.ATTN_TP:
|
|
return parallel.attn_tp_group
|
|
if group is SumGroup.TP:
|
|
return parallel.tp_group
|
|
return post_experts_reduction_group()
|
|
|
|
|
|
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
|
"""How the rows a layer takes become its attention's input."""
|
|
if rows == need.layout:
|
|
return CommunicateSimpleFn._trivial
|
|
raise NotImplementedError(f"{rows=} {need=}")
|
|
|
|
|
|
def _select_ffn_input(
|
|
produced: StageOutput,
|
|
*,
|
|
residual: Layout,
|
|
need: StageInput,
|
|
force_layernorm_before_gather: bool,
|
|
) -> Callable:
|
|
"""The steps from the attention output to the FFN input: complete the
|
|
attention-TP sum, add the residual and normalize, and gather the rows the
|
|
FFN's group needs, with the residual left on the attention's rows."""
|
|
gathered = produced.layout.sharded - need.layout.sharded
|
|
if (
|
|
residual != produced.layout
|
|
or not need.layout.sharded <= produced.layout.sharded
|
|
or gathered != {TokenAxis.ATTN_DP}
|
|
):
|
|
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
|
# What the attention output owes decides the steps: the attention-TP sum,
|
|
# always left by the output projection, or nothing.
|
|
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
|
if owes_attention_tp != produced.always_leaves or produced.group not in (
|
|
None,
|
|
SumGroup.ATTN_TP,
|
|
):
|
|
raise NotImplementedError(f"{produced=}")
|
|
# The partial order adds the residual on attention-TP rank 0 before the DP
|
|
# gather's collective completes that sum, which only a plain residual add
|
|
# allows.
|
|
if owes_attention_tp and not force_layernorm_before_gather:
|
|
return partial(_mlp_input_dp_partial, gathers_residual=False)
|
|
return partial(
|
|
_mlp_input_dp_replicate,
|
|
gathers_residual=False,
|
|
reduces_attention_tp=owes_attention_tp,
|
|
)
|
|
|
|
|
|
def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool:
|
|
"""Whether the FFN output goes back to the layer's rows by undoing the
|
|
attention-DP gather."""
|
|
returned = rows.sharded - produced.layout.sharded
|
|
if not produced.layout.sharded <= rows.sharded or returned not in (
|
|
frozenset(),
|
|
{TokenAxis.ATTN_DP},
|
|
):
|
|
raise NotImplementedError(f"{produced=} {rows=}")
|
|
return bool(returned)
|
|
|
|
|
|
class MlpInputKind(Enum):
|
|
"""What the attention-TP -> FFN boundary does, given the two sides' layouts."""
|
|
|
|
# The layouts agree: add the residual and normalize.
|
|
NORM = auto()
|
|
# All-reduce over attention TP, normalize, and gather for the FFN group.
|
|
GATHER = auto()
|
|
# The same, then gather over the MoE-CP group (moe_dp_size < attn_cp_size).
|
|
GATHER_MOE_CP = auto()
|
|
# Reduce-scatter over attention TP to this rank's tokens, then normalize.
|
|
SCATTER = auto()
|
|
# All-reduce over attention TP for a dense MLP on that group.
|
|
ATTN_TP_ALL_REDUCE = auto()
|
|
|
|
|
|
def mlp_input_kind(
|
|
modes: LayerScatterModes, context: CommunicateContext
|
|
) -> MlpInputKind:
|
|
hidden_in, residual_in = modes.attn_mode, modes.layer_input_mode
|
|
hidden_out, residual_out = modes.mlp_mode, modes.middle_residual_mode
|
|
if (
|
|
context.is_same_layout(hidden_in, hidden_out)
|
|
and context.is_same_layout(residual_in, residual_out)
|
|
and context.attn_tp_size == 1
|
|
):
|
|
return MlpInputKind.NORM
|
|
if (
|
|
hidden_in == ScatterMode.SCATTERED
|
|
and residual_in == ScatterMode.SCATTERED
|
|
and hidden_out == ScatterMode.SCATTERED
|
|
and residual_out == ScatterMode.SCATTERED
|
|
):
|
|
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
|
|
# activations stay sequence-sharded across the attn->mlp boundary, so
|
|
# there is nothing to gather or scatter here -- just the residual add
|
|
# plus LayerNorm on the local shard. The row-parallel o_proj already
|
|
# issued the reduce-scatter (g-bar) that the all-reduce would have
|
|
# done, and the g all-gather is fused into the next column-parallel
|
|
# linear. Distinct from the branch above because under pure TP
|
|
# attn_tp_size == tp_size > 1, so that gate does not fire.
|
|
return MlpInputKind.NORM
|
|
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
):
|
|
kind = {
|
|
(ScatterMode.FULL, ScatterMode.TP_ATTN_FULL): MlpInputKind.GATHER,
|
|
(
|
|
ScatterMode.MOE_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): MlpInputKind.GATHER_MOE_CP,
|
|
(ScatterMode.SCATTERED, ScatterMode.SCATTERED): MlpInputKind.SCATTER,
|
|
}.get((hidden_out, residual_out))
|
|
if kind is not None:
|
|
return kind
|
|
if (
|
|
hidden_out == ScatterMode.TP_ATTN_FULL
|
|
and residual_out == ScatterMode.TP_ATTN_FULL
|
|
and context.attn_tp_size > 1
|
|
):
|
|
# Used when the dense MLP is tensor-parallelized along the
|
|
# attention TP group (``moe_dense_tp_size > 1``): hidden states
|
|
# need an all-reduce inside the attention TP group before the
|
|
# next layernorm, while staying in TP_ATTN_FULL on both sides.
|
|
return MlpInputKind.ATTN_TP_ALL_REDUCE
|
|
raise NotImplementedError(
|
|
f"{hidden_in=} {residual_in=} {hidden_out=} {residual_out=}"
|
|
)
|
|
|
|
|
|
def _mlp_input_norm(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
# TODO move these `if shape != 0` into LayerNorm itself
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
|
|
def _mlp_input_attn_tp_all_reduce(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
"""All-reduce hidden states inside the attention TP group, then layernorm.
|
|
|
|
Used when the dense MLP shares the attention TP group
|
|
(``moe_dense_tp_size > 1``): both hidden states and residual stay in
|
|
``TP_ATTN_FULL`` across the boundary.
|
|
"""
|
|
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
|
|
def _mlp_input_scatter(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
scatters_residual: bool,
|
|
):
|
|
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
|
hidden_states, context
|
|
)
|
|
if scatters_residual:
|
|
residual = _redistribute_to_attn_tp_shards(residual, context)
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states, residual = layernorm(hidden_states, residual)
|
|
return hidden_states, residual
|
|
|
|
|
|
def _tp_all_reduce_with_scattered_residual(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
):
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, hidden_states
|
|
|
|
scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
|
scattered_states += residual
|
|
residual = tensor_model_parallel_all_reduce(hidden_states)
|
|
hidden_states = layernorm(residual)
|
|
return hidden_states, residual
|
|
|
|
|
|
def _mlp_input_gather(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
order: Callable,
|
|
):
|
|
"""Run ``order``, the steps ``_mlp_input_order`` chose at construction,
|
|
unless this batch's attention input is scattered."""
|
|
if get_attn_tp_context().input_scattered:
|
|
return _tp_all_reduce_with_scattered_residual(
|
|
hidden_states, residual, layernorm, context
|
|
)
|
|
return order(hidden_states, residual, forward_batch, layernorm, context)
|
|
|
|
|
|
def _mlp_input_gather_moe_cp(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
layernorm: torch.nn.Module,
|
|
context: CommunicateContext,
|
|
*,
|
|
gather: Callable,
|
|
):
|
|
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
|
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
|
TP_ATTN_FULL."""
|
|
# Early return on empty tensor is safe for MOE_CP because:
|
|
# - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens,
|
|
# so no rank hits this path while others proceed to the allgather.
|
|
# - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend).
|
|
# - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used.
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, residual
|
|
|
|
hidden_states, residual = gather(
|
|
hidden_states, residual, forward_batch, layernorm, context
|
|
)
|
|
|
|
# Only active during prefill (context-parallel extend); decode keeps existing path.
|
|
moe_cp_size = get_moe_cp_size()
|
|
if (
|
|
moe_cp_size > 1
|
|
and hidden_states.shape[0] > 0
|
|
and forward_batch.forward_mode.is_context_parallel_extend()
|
|
and forward_batch.attn_cp_metadata is not None
|
|
):
|
|
hidden_states = _redistribute_input_to_moe_cp(
|
|
hidden_states, forward_batch, moe_cp_size
|
|
)
|
|
|
|
return hidden_states, residual
|
|
|
|
|
|
def _dp_scatter_group() -> GroupCoordinator:
|
|
parallel = get_parallel()
|
|
if parallel.tp_size == parallel.attn_dp_size:
|
|
return parallel.tp_group
|
|
return parallel.attn_tp_group
|
|
|
|
|
|
def _reduce_and_redistribute_output_varlen(
|
|
local_hidden_states: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> None:
|
|
get_parallel().tp_group.reduce_scatterv(
|
|
hidden_states,
|
|
output=local_hidden_states,
|
|
sizes=get_dp_global_num_tokens(),
|
|
)
|
|
|
|
|
|
def _reduce_and_redistribute_output_max_len(
|
|
local_hidden_states: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> None:
|
|
dp_reduce_scatter_tensor(local_hidden_states, hidden_states)
|
|
|
|
|
|
def _redistribute_output(
|
|
local_hidden_states: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> None:
|
|
dp_scatter(local_hidden_states, hidden_states, forward_batch)
|
|
|
|
|
|
def _redistribute_output_from_moe_cp(
|
|
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
|
) -> torch.Tensor:
|
|
moe_cp_rank = get_moe_cp_rank()
|
|
# The allgather was padded to max_tokens_per_rank (equal chunks).
|
|
# Extract this rank's actual (non-padded) tokens from its chunk.
|
|
per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token
|
|
max_tokens_per_rank = max(per_rank_tokens)
|
|
actual_local_tokens = per_rank_tokens[moe_cp_rank]
|
|
return hidden_states.narrow(
|
|
0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens
|
|
).contiguous()
|
|
|
|
|
|
def _reduce_and_redistribute_output_step(
|
|
forward_batch: ForwardBatch,
|
|
*,
|
|
leaves_for_reduce_scatter: bool,
|
|
leaves_for_reduce_scatterv: bool,
|
|
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
|
"""The reduce-scatter that brings an FFN output gathered over attention
|
|
DP back to this rank's tokens when the FFN leaves its sum to it (see
|
|
StageOutput); None when the FFN reduces the output and only a scatter
|
|
remains."""
|
|
if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv:
|
|
return _reduce_and_redistribute_output_varlen
|
|
if (
|
|
leaves_for_reduce_scatter
|
|
and forward_batch.dp_padding_mode.is_max_len()
|
|
and can_use_dp_reduce_scatter()
|
|
):
|
|
return _reduce_and_redistribute_output_max_len
|
|
return None
|
|
|
|
|
|
def _to_local_tokens(
|
|
step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None],
|
|
forward_batch: ForwardBatch,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
local_hidden_states = get_local_dp_buffer(_dp_scatter_group())
|
|
step(local_hidden_states, hidden_states, forward_batch)
|
|
return local_hidden_states
|
|
|
|
|
|
def _all_reduce_then_to_local_tokens(
|
|
group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor
|
|
) -> torch.Tensor:
|
|
return _to_local_tokens(
|
|
_redistribute_output, forward_batch, group.all_reduce(hidden_states)
|
|
)
|
|
|
|
|
|
class CommunicateSummableTensorPairFn:
|
|
"""It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed."""
|
|
|
|
@staticmethod
|
|
def get_fn(
|
|
hidden_states_input_mode: ScatterMode,
|
|
residual_input_mode: ScatterMode,
|
|
output_mode: ScatterMode,
|
|
context: CommunicateContext,
|
|
):
|
|
if context.is_same_layout(
|
|
hidden_states_input_mode, output_mode
|
|
) and context.is_same_layout(residual_input_mode, output_mode):
|
|
return CommunicateSummableTensorPairFn._trivial
|
|
|
|
fn = {
|
|
(
|
|
ScatterMode.FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._scatter_hidden_states,
|
|
(
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.SCATTERED,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._gather,
|
|
(
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.SCATTERED,
|
|
): CommunicateSummableTensorPairFn._scatter,
|
|
(
|
|
ScatterMode.MOE_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
ScatterMode.TP_ATTN_FULL,
|
|
): CommunicateSummableTensorPairFn._scatter_hidden_states_moe,
|
|
}.get((hidden_states_input_mode, residual_input_mode, output_mode))
|
|
if fn is not None:
|
|
return fn
|
|
|
|
raise NotImplementedError(
|
|
f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}"
|
|
)
|
|
|
|
@staticmethod
|
|
def _trivial(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
return hidden_states, residual
|
|
|
|
@staticmethod
|
|
def _scatter_hidden_states(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
allow_reduce_scatter: bool = False,
|
|
is_layer_sparse: bool = False,
|
|
):
|
|
step = (
|
|
_reduce_and_redistribute_output_step(
|
|
forward_batch,
|
|
leaves_for_reduce_scatter=allow_reduce_scatter,
|
|
# A MoE block leaves its sum to reduce_scatterv whenever it
|
|
# applies (should_skip_post_experts_all_reduce).
|
|
leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse,
|
|
)
|
|
or _redistribute_output
|
|
)
|
|
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
|
|
|
@staticmethod
|
|
def _gather(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
hidden_states += residual
|
|
return _redistribute_from_attn_tp_shards(hidden_states), None
|
|
|
|
@staticmethod
|
|
def _scatter(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
):
|
|
assert residual is None, "not yet handled residual!=None"
|
|
return _redistribute_to_attn_tp_shards(hidden_states, context), None
|
|
|
|
@staticmethod
|
|
def _scatter_hidden_states_moe(
|
|
hidden_states: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
context: CommunicateContext,
|
|
**kwargs,
|
|
):
|
|
"""Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation.
|
|
|
|
After moe_tensor_model_parallel_all_reduce (which runs unconditionally since
|
|
mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the
|
|
full MoE result for all cp_per_moe token chunks. We simply slice out this rank's
|
|
CP-local portion.
|
|
|
|
If DP>1, further scatter back to the local DP slice.
|
|
"""
|
|
# Only scatter back during prefill; decode was never allgathered so no-op.
|
|
# Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp
|
|
# — CP extend always has non-zero tokens per rank, and decode skips this path.
|
|
moe_cp_size = get_moe_cp_size()
|
|
if (
|
|
moe_cp_size > 1
|
|
and forward_batch.forward_mode.is_context_parallel_extend()
|
|
and forward_batch.attn_cp_metadata is not None
|
|
):
|
|
hidden_states = _redistribute_output_from_moe_cp(
|
|
hidden_states, forward_batch
|
|
)
|
|
|
|
if context.attn_dp_size > 1:
|
|
hidden_states = _to_local_tokens(
|
|
_redistribute_output, forward_batch, hidden_states
|
|
)
|
|
|
|
return hidden_states, residual
|