[Refactor] Carry a deferred FFN all-reduce as UnreducedOutput and complete it in the next layer without the fused kernel (#41196)
This commit is contained in:
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
|
||||
CommunicateContext,
|
||||
CommunicateSummableTensorPairFn,
|
||||
ScatterMode,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
@@ -966,6 +967,7 @@ def _model_forward_tbo(
|
||||
input_data_scatter_mode: ScatterMode,
|
||||
layer_input_scatter_mode: ScatterMode,
|
||||
):
|
||||
inputs["hidden_states"] = reduce_output(inputs["hidden_states"])
|
||||
inputs_arr = _model_forward_tbo_split_inputs(
|
||||
**inputs,
|
||||
input_data_scatter_mode=input_data_scatter_mode,
|
||||
|
||||
@@ -18,6 +18,7 @@ from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Dict, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
@@ -66,6 +67,7 @@ from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
deferred_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
post_experts_output_is_complete,
|
||||
should_use_dp_reduce_scatterv,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
@@ -82,6 +84,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_spec,
|
||||
@@ -621,6 +624,68 @@ def _attn_input_update_and_read_residual(quant_format: str):
|
||||
return _update_and_read_residual_plain
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its FFN all-reduce, left for the next
|
||||
layer's input norm. Hand it to the next layer, or pass it through
|
||||
reduce_output() before reading it any other way."""
|
||||
|
||||
partial: torch.Tensor
|
||||
|
||||
|
||||
def reduce_output(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Run the all-reduce an UnreducedOutput still owes; pass anything else through."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return deferred_post_experts_all_reduce(hidden_states.partial)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def layer_input_buffer(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
) -> torch.Tensor:
|
||||
"""The tensor holding a layer's input, for reusing its memory without reading it."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return hidden_states.partial
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _batch_size(forward_batch: ForwardBatch) -> int:
|
||||
return (
|
||||
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
|
||||
)
|
||||
|
||||
|
||||
def _deferred_reduction_runs_on_the_tp_group() -> bool:
|
||||
"""Whether deferred_post_experts_all_reduce reduces over the TP group object
|
||||
itself, the group a dense FFN and reduce_moe_output reduce over."""
|
||||
parallel = get_parallel()
|
||||
if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1:
|
||||
# Some MoE blocks reduce EP and MoE-TP in two steps instead of merging.
|
||||
return False
|
||||
group = parallel.moe_ep_group if parallel.moe_ep_size > 1 else parallel.moe_tp_group
|
||||
return group is parallel.tp_group
|
||||
|
||||
|
||||
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether deferred_post_experts_all_reduce is the all-reduce the FFN would
|
||||
have run itself: one full-precision sum over the same TP group, on an output
|
||||
that is a plain partial sum, with no attention DP layout to restore. The
|
||||
output is not a plain partial sum when the MoE combine already summed it, a
|
||||
replicated shared expert is added after the reduction, or LoRA-B runs on
|
||||
the unreduced activations."""
|
||||
return (
|
||||
_batch_size(forward_batch) > 0
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not post_experts_output_is_complete(is_tp_path=True)
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
and not envs.SGLANG_SHARED_EXPERT_TP1.get()
|
||||
and not get_lora().enable_lora
|
||||
and _deferred_reduction_runs_on_the_tp_group()
|
||||
)
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -762,12 +827,15 @@ class LayerCommunicator:
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
pending = isinstance(hidden_states, UnreducedOutput)
|
||||
if pending:
|
||||
hidden_states = hidden_states.partial
|
||||
# residual is None marks the first decoder layer, where the SP region
|
||||
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
|
||||
# the next one.
|
||||
@@ -793,14 +861,21 @@ class LayerCommunicator:
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
residual = hidden_states
|
||||
elif (
|
||||
residual is not None
|
||||
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
hidden_states, residual = self._reduce_output_and_update_and_read_residual(
|
||||
hidden_states, residual, forward_batch
|
||||
elif residual is not None and pending:
|
||||
fused = (
|
||||
self._reduce_output_and_update_and_read_residual(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if post_residual_addition is None
|
||||
else None
|
||||
)
|
||||
if fused is not None:
|
||||
hidden_states, residual = fused
|
||||
else:
|
||||
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
||||
else:
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
@@ -831,12 +906,10 @@ class LayerCommunicator:
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Complete the sum the previous layer left, add it to the residual and
|
||||
apply the input norm:
|
||||
fused into one kernel when it takes this batch, else as an all-reduce
|
||||
followed by the norm. Neither applies ``quant_format`` or
|
||||
``post_residual_addition``."""
|
||||
apply the input norm in one fused kernel; None when the kernel does not
|
||||
take this batch. The result is not quantized for ``quant_format``."""
|
||||
if (
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
@@ -864,8 +937,7 @@ class LayerCommunicator:
|
||||
return self.input_layernorm.forward_with_allreduce_fusion(
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
||||
return self.input_layernorm(hidden_states, residual)
|
||||
return None
|
||||
|
||||
def _tp_reduce_scatter(
|
||||
self,
|
||||
@@ -929,14 +1001,14 @@ class LayerCommunicator:
|
||||
|
||||
def finish_layer_stack(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Complete what this layer left for a next layer. Call it on the last
|
||||
layer of this rank before its output reaches the final norm, the next
|
||||
pipeline rank, or any other consumer outside the layers."""
|
||||
return complete_deferred_allreduce(hidden_states), residual
|
||||
return reduce_output(hidden_states), residual
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
@@ -967,10 +1039,7 @@ class LayerCommunicator:
|
||||
"""Whether the MoE may hand an unfinalized output to the next layer."""
|
||||
return False
|
||||
|
||||
# NOTE: This function will cause torch recompilation
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
def _ffn_sum_can_move_to_next_layer(self) -> bool:
|
||||
# When MOE_FULL is active (moe_cp allgather), fusion must be disabled because
|
||||
# the fusion path skips postprocess_layer which contains the moe_cp scatter.
|
||||
# Without scatter, hidden_states remain at MOE_FULL size while residual is at
|
||||
@@ -1003,17 +1072,17 @@ class LayerCommunicator:
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return False
|
||||
|
||||
batch_size = (
|
||||
forward_batch.input_ids.shape[0]
|
||||
if hasattr(forward_batch, "input_ids")
|
||||
else 0
|
||||
)
|
||||
|
||||
# When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP
|
||||
# all-reduce, so there is nothing to fuse with the next layer.
|
||||
if self.layer_scatter_modes.mlp_mode == ScatterMode.SCATTERED:
|
||||
return False
|
||||
return self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED
|
||||
|
||||
# NOTE: This function will cause torch recompilation
|
||||
def should_fuse_mlp_allreduce_with_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if not self._ffn_sum_can_move_to_next_layer():
|
||||
return False
|
||||
batch_size = _batch_size(forward_batch)
|
||||
return (
|
||||
(
|
||||
apply_flashinfer_allreduce_fusion(batch_size)
|
||||
@@ -1030,6 +1099,20 @@ class LayerCommunicator:
|
||||
and (self._context.tp_size > 1)
|
||||
)
|
||||
|
||||
def should_defer_ffn_reduction(self, forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
input norm: whenever the fused kernel takes it, and otherwise when the
|
||||
next layer would run the same all-reduce the FFN itself would have."""
|
||||
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
|
||||
return True
|
||||
return (
|
||||
self._context.tp_size > 1
|
||||
and not self.is_last_layer
|
||||
and self._ffn_sum_can_move_to_next_layer()
|
||||
and _unfused_completion_matches_the_ffn(forward_batch)
|
||||
and not self.should_use_reduce_scatter(forward_batch)
|
||||
)
|
||||
|
||||
|
||||
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
|
||||
_SCATTER_MODE_SHARDED_AXES = {
|
||||
@@ -1079,7 +1162,7 @@ class FfnExit:
|
||||
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
||||
self.fuse_mlp_allreduce = (
|
||||
self.defer_moe_finalize
|
||||
or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
or communicator.should_defer_ffn_reduction(forward_batch)
|
||||
)
|
||||
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
|
||||
self._scope = get_forward().scoped(
|
||||
@@ -1097,31 +1180,19 @@ class FfnExit:
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input norm, or postprocess."""
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
return hidden_states, residual
|
||||
if self.fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states, residual
|
||||
return UnreducedOutput(hidden_states), residual
|
||||
return self.communicator.postprocess_layer(
|
||||
hidden_states, residual, self.forward_batch
|
||||
)
|
||||
|
||||
|
||||
def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Run the all-reduce a layer left for the next layer's input norm."""
|
||||
if (
|
||||
hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
return hidden_states
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
|
||||
@@ -558,6 +558,9 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch):
|
||||
return False
|
||||
|
||||
def should_defer_ffn_reduction(self, forward_batch):
|
||||
return False
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
return False
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
is_tbo_enabled,
|
||||
post_experts_all_reduce,
|
||||
post_experts_output_is_complete,
|
||||
reduce_moe_output,
|
||||
should_skip_mlp_all_reduce,
|
||||
should_skip_post_experts_all_reduce,
|
||||
@@ -35,6 +36,7 @@ __all__ = [
|
||||
"can_merge_post_experts_all_reduce",
|
||||
"deferred_post_experts_all_reduce",
|
||||
"post_experts_all_reduce",
|
||||
"post_experts_output_is_complete",
|
||||
"reduce_moe_output",
|
||||
"should_skip_post_experts_all_reduce",
|
||||
"should_use_dp_reduce_scatterv",
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
|
||||
CommunicateWithAllReduceAndLayerNormFn,
|
||||
LayerCommunicator,
|
||||
ScatterMode,
|
||||
UnreducedOutput,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -221,17 +222,16 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
|
||||
if (
|
||||
residual is not None
|
||||
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
and isinstance(hidden_states, UnreducedOutput)
|
||||
and self._can_consume_post_moe_all_reduce(
|
||||
forward_batch, int(hidden_states.shape[0])
|
||||
forward_batch, int(hidden_states.partial.shape[0])
|
||||
)
|
||||
):
|
||||
if post_residual_addition is not None:
|
||||
residual = residual + post_residual_addition
|
||||
assert self.fusion_service is not None
|
||||
hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm(
|
||||
local_contribution=hidden_states,
|
||||
local_contribution=hidden_states.partial,
|
||||
residual=residual,
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
)
|
||||
|
||||
@@ -34,8 +34,8 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -828,7 +828,7 @@ class BailingMoEModel(nn.Module):
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
|
||||
@@ -31,8 +31,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -1385,9 +1386,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
if (
|
||||
capture_aux
|
||||
and i in self.layers_to_capture
|
||||
and hidden_states.shape[0] != 0
|
||||
and layer_input_buffer(hidden_states).shape[0] != 0
|
||||
):
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
if residual is None:
|
||||
dspark_aux_hidden_states.append(hidden_states)
|
||||
else:
|
||||
|
||||
@@ -72,6 +72,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerScatterModes,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
)
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
DSACPLayerCommunicator,
|
||||
@@ -2688,7 +2689,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
||||
next_full_attention_layer_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_orig = hidden_states
|
||||
hidden_states_orig = layer_input_buffer(hidden_states)
|
||||
hidden_states, residual = (
|
||||
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
|
||||
hidden_states,
|
||||
|
||||
@@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
UnreducedOutput,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1638,7 +1639,7 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
if should_allreduce_fusion:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
|
||||
if not should_allreduce_fusion:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
|
||||
@@ -41,8 +41,8 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
@@ -1109,7 +1109,7 @@ class Glm4MoeModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
|
||||
@@ -35,9 +35,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
@@ -804,7 +804,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
for i in range(normal_start_layer, normal_end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
|
||||
@@ -28,9 +28,11 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
UnreducedOutput,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
@@ -923,7 +925,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
prev_topk_indices: Optional[torch.Tensor] = None,
|
||||
):
|
||||
hidden_states_orig = hidden_states
|
||||
hidden_states_orig = layer_input_buffer(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states,
|
||||
@@ -987,7 +989,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
if should_allreduce_fusion:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
|
||||
if not should_allreduce_fusion:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
@@ -1182,7 +1184,7 @@ class Glm5NextModel(nn.Module):
|
||||
)
|
||||
with ctx:
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_state = self._prepare_aux_hidden_state(
|
||||
hidden_states, residual
|
||||
)
|
||||
|
||||
@@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -725,7 +725,7 @@ class GptOssModel(nn.Module):
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
if i + 1 in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual
|
||||
if residual is not None
|
||||
|
||||
@@ -22,7 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -592,7 +592,7 @@ class LagunaModel(nn.Module):
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
UnreducedOutput,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1417,7 +1418,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
if should_allreduce_fusion:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
|
||||
@@ -25,15 +25,13 @@ from torch import nn
|
||||
from sglang.srt.compilation.compilation_config import register_split_op
|
||||
from sglang.srt.configs import NemotronHConfig
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.activation import ReLU2
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
HybridLinearAttnBackend,
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import UnreducedOutput, reduce_output
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
@@ -429,7 +427,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
):
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
@@ -533,7 +531,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
hidden_states, forward_batch, skip_reduce
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -810,11 +808,7 @@ class NemotronHModel(nn.Module):
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
if residual is not None and getattr(
|
||||
hidden_states, "_sglang_needs_allreduce_fusion", False
|
||||
):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
self._capture_hidden_states(hidden_states, residual, i)
|
||||
)
|
||||
|
||||
@@ -46,7 +46,8 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -312,8 +313,7 @@ def _finish_mlp_output(hidden_states, *, expect_deferred: bool):
|
||||
if not expect_deferred:
|
||||
if isinstance(hidden_states, MoeFinalizeHandoff):
|
||||
raise RuntimeError("unexpected deferred-finalize handoff")
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states
|
||||
return UnreducedOutput(hidden_states)
|
||||
|
||||
if not isinstance(hidden_states, MoeFinalizeHandoff):
|
||||
raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff")
|
||||
@@ -1938,7 +1938,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
and layer_idx < 3
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
)
|
||||
|
||||
@@ -24,7 +24,7 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import complete_deferred_allreduce
|
||||
from sglang.srt.layers.communicator import reduce_output
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
@@ -107,7 +107,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
):
|
||||
layer_idx += self.start_layer
|
||||
if layer_idx in self.layers_to_capture:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
)
|
||||
@@ -140,7 +140,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
and layer_idx in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states.add_(
|
||||
input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
)
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
UnreducedOutput,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -641,7 +642,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
fuse_mlp_allreduce = False
|
||||
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
hidden_states = UnreducedOutput(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
|
||||
Reference in New Issue
Block a user