[Refactor] Carry a deferred FFN all-reduce as UnreducedOutput and complete it in the next layer without the fused kernel (#41196)

This commit is contained in:
Cheng Wan
2026-09-25 01:46:03 -07:00
committed by GitHub
parent 402df23188
commit 9d7f44bdbb
27 changed files with 426 additions and 165 deletions
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
CommunicateContext,
CommunicateSummableTensorPairFn,
ScatterMode,
reduce_output,
)
from sglang.srt.layers.moe import (
get_deepep_mode,
@@ -966,6 +967,7 @@ def _model_forward_tbo(
input_data_scatter_mode: ScatterMode,
layer_input_scatter_mode: ScatterMode,
):
inputs["hidden_states"] = reduce_output(inputs["hidden_states"])
inputs_arr = _model_forward_tbo_split_inputs(
**inputs,
input_data_scatter_mode=input_data_scatter_mode,
+115 -44
View File
@@ -18,6 +18,7 @@ from enum import Enum, auto
from functools import cached_property, partial
from typing import Callable, Dict, Optional, Tuple, Union
import msgspec
import torch
from sglang.srt.distributed import (
@@ -66,6 +67,7 @@ from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
deferred_post_experts_all_reduce,
get_moe_a2a_backend,
post_experts_output_is_complete,
should_use_dp_reduce_scatterv,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
@@ -82,6 +84,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_parallel,
get_platform,
get_spec,
@@ -621,6 +624,68 @@ def _attn_input_update_and_read_residual(quant_format: str):
return _update_and_read_residual_plain
class UnreducedOutput(msgspec.Struct, frozen=True):
"""A layer output that still owes its FFN all-reduce, left for the next
layer's input norm. Hand it to the next layer, or pass it through
reduce_output() before reading it any other way."""
partial: torch.Tensor
def reduce_output(
hidden_states: Union[torch.Tensor, UnreducedOutput, None],
) -> Optional[torch.Tensor]:
"""Run the all-reduce an UnreducedOutput still owes; pass anything else through."""
if isinstance(hidden_states, UnreducedOutput):
return deferred_post_experts_all_reduce(hidden_states.partial)
return hidden_states
def layer_input_buffer(
hidden_states: Union[torch.Tensor, UnreducedOutput],
) -> torch.Tensor:
"""The tensor holding a layer's input, for reusing its memory without reading it."""
if isinstance(hidden_states, UnreducedOutput):
return hidden_states.partial
return hidden_states
def _batch_size(forward_batch: ForwardBatch) -> int:
return (
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
)
def _deferred_reduction_runs_on_the_tp_group() -> bool:
"""Whether deferred_post_experts_all_reduce reduces over the TP group object
itself, the group a dense FFN and reduce_moe_output reduce over."""
parallel = get_parallel()
if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1:
# Some MoE blocks reduce EP and MoE-TP in two steps instead of merging.
return False
group = parallel.moe_ep_group if parallel.moe_ep_size > 1 else parallel.moe_tp_group
return group is parallel.tp_group
def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
"""Whether deferred_post_experts_all_reduce is the all-reduce the FFN would
have run itself: one full-precision sum over the same TP group, on an output
that is a plain partial sum, with no attention DP layout to restore. The
output is not a plain partial sum when the MoE combine already summed it, a
replicated shared expert is added after the reduction, or LoRA-B runs on
the unreduced activations."""
return (
_batch_size(forward_batch) > 0
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and not post_experts_output_is_complete(is_tp_path=True)
and not get_exec().comm.enable_quant_communications
and not envs.SGLANG_SHARED_EXPERT_TP1.get()
and not get_lora().enable_lora
and _deferred_reduction_runs_on_the_tp_group()
)
class LayerCommunicator:
def __init__(
self,
@@ -762,12 +827,15 @@ class LayerCommunicator:
def prepare_attn(
self,
hidden_states: torch.Tensor,
hidden_states: Union[torch.Tensor, UnreducedOutput],
residual: torch.Tensor,
forward_batch: ForwardBatch,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
pending = isinstance(hidden_states, UnreducedOutput)
if pending:
hidden_states = hidden_states.partial
# residual is None marks the first decoder layer, where the SP region
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
# the next one.
@@ -793,14 +861,21 @@ class LayerCommunicator:
)
if hidden_states.shape[0] == 0:
residual = hidden_states
elif (
residual is not None
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
):
hidden_states, residual = self._reduce_output_and_update_and_read_residual(
hidden_states, residual, forward_batch
elif residual is not None and pending:
fused = (
self._reduce_output_and_update_and_read_residual(
hidden_states, residual, forward_batch
)
if post_residual_addition is None
else None
)
if fused is not None:
hidden_states, residual = fused
else:
hidden_states = deferred_post_experts_all_reduce(hidden_states)
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
else:
hidden_states, residual = _attn_input_update_and_read_residual(
quant_format
@@ -831,12 +906,10 @@ class LayerCommunicator:
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""Complete the sum the previous layer left, add it to the residual and
apply the input norm:
fused into one kernel when it takes this batch, else as an all-reduce
followed by the norm. Neither applies ``quant_format`` or
``post_residual_addition``."""
apply the input norm in one fused kernel; None when the kernel does not
take this batch. The result is not quantized for ``quant_format``."""
if (
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
@@ -864,8 +937,7 @@ class LayerCommunicator:
return self.input_layernorm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=False
)
hidden_states = deferred_post_experts_all_reduce(hidden_states)
return self.input_layernorm(hidden_states, residual)
return None
def _tp_reduce_scatter(
self,
@@ -929,14 +1001,14 @@ class LayerCommunicator:
def finish_layer_stack(
self,
hidden_states: torch.Tensor,
hidden_states: Union[torch.Tensor, UnreducedOutput],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer. Call it on the last
layer of this rank before its output reaches the final norm, the next
pipeline rank, or any other consumer outside the layers."""
return complete_deferred_allreduce(hidden_states), residual
return reduce_output(hidden_states), residual
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
if not self.allow_reduce_scatter:
@@ -967,10 +1039,7 @@ class LayerCommunicator:
"""Whether the MoE may hand an unfinalized output to the next layer."""
return False
# NOTE: This function will cause torch recompilation
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
def _ffn_sum_can_move_to_next_layer(self) -> bool:
# When MOE_FULL is active (moe_cp allgather), fusion must be disabled because
# the fusion path skips postprocess_layer which contains the moe_cp scatter.
# Without scatter, hidden_states remain at MOE_FULL size while residual is at
@@ -1003,17 +1072,17 @@ class LayerCommunicator:
if get_attn_tp_context().input_scattered:
return False
batch_size = (
forward_batch.input_ids.shape[0]
if hasattr(forward_batch, "input_ids")
else 0
)
# When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP
# all-reduce, so there is nothing to fuse with the next layer.
if self.layer_scatter_modes.mlp_mode == ScatterMode.SCATTERED:
return False
return self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED
# NOTE: This function will cause torch recompilation
def should_fuse_mlp_allreduce_with_next_layer(
self, forward_batch: ForwardBatch
) -> bool:
if not self._ffn_sum_can_move_to_next_layer():
return False
batch_size = _batch_size(forward_batch)
return (
(
apply_flashinfer_allreduce_fusion(batch_size)
@@ -1030,6 +1099,20 @@ class LayerCommunicator:
and (self._context.tp_size > 1)
)
def should_defer_ffn_reduction(self, forward_batch: ForwardBatch) -> bool:
"""Whether the FFN leaves its output's all-reduce to the next layer's
input norm: whenever the fused kernel takes it, and otherwise when the
next layer would run the same all-reduce the FFN itself would have."""
if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch):
return True
return (
self._context.tp_size > 1
and not self.is_last_layer
and self._ffn_sum_can_move_to_next_layer()
and _unfused_completion_matches_the_ffn(forward_batch)
and not self.should_use_reduce_scatter(forward_batch)
)
# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on.
_SCATTER_MODE_SHARDED_AXES = {
@@ -1079,7 +1162,7 @@ class FfnExit:
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
self.fuse_mlp_allreduce = (
self.defer_moe_finalize
or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
or communicator.should_defer_ffn_reduction(forward_batch)
)
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
self._scope = get_forward().scoped(
@@ -1097,31 +1180,19 @@ class FfnExit:
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
"""Leave the reduction to the next layer's input norm, or postprocess."""
if not isinstance(hidden_states, torch.Tensor):
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
return hidden_states, residual
if self.fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states, residual
return UnreducedOutput(hidden_states), residual
return self.communicator.postprocess_layer(
hidden_states, residual, self.forward_batch
)
def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor:
"""Run the all-reduce a layer left for the next layer's input norm."""
if (
hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
):
hidden_states = deferred_post_experts_all_reduce(hidden_states)
hidden_states._sglang_needs_allreduce_fusion = False
return hidden_states
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
@@ -558,6 +558,9 @@ class MHCLayerCommunicator(LayerCommunicator):
def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch):
return False
def should_defer_ffn_reduction(self, forward_batch):
return False
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
if not self.allow_reduce_scatter:
return False
+2
View File
@@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import (
is_moe_input_scattered_across_dp_ranks,
is_tbo_enabled,
post_experts_all_reduce,
post_experts_output_is_complete,
reduce_moe_output,
should_skip_mlp_all_reduce,
should_skip_post_experts_all_reduce,
@@ -35,6 +36,7 @@ __all__ = [
"can_merge_post_experts_all_reduce",
"deferred_post_experts_all_reduce",
"post_experts_all_reduce",
"post_experts_output_is_complete",
"reduce_moe_output",
"should_skip_post_experts_all_reduce",
"should_use_dp_reduce_scatterv",
@@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import (
CommunicateWithAllReduceAndLayerNormFn,
LayerCommunicator,
ScatterMode,
UnreducedOutput,
get_attn_tp_context,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -221,17 +222,16 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
if (
residual is not None
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
and isinstance(hidden_states, UnreducedOutput)
and self._can_consume_post_moe_all_reduce(
forward_batch, int(hidden_states.shape[0])
forward_batch, int(hidden_states.partial.shape[0])
)
):
if post_residual_addition is not None:
residual = residual + post_residual_addition
assert self.fusion_service is not None
hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm(
local_contribution=hidden_states,
local_contribution=hidden_states.partial,
residual=residual,
gamma=_fused_norm_gamma(self.input_layernorm),
)
+2 -2
View File
@@ -34,8 +34,8 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -828,7 +828,7 @@ class BailingMoEModel(nn.Module):
for i in range(self.start_layer, self.end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states if residual is None else hidden_states + residual
)
+4 -3
View File
@@ -31,8 +31,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
layer_input_buffer,
reduce_output,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
@@ -1385,9 +1386,9 @@ class BailingMoELinearModel(nn.Module):
if (
capture_aux
and i in self.layers_to_capture
and hidden_states.shape[0] != 0
and layer_input_buffer(hidden_states).shape[0] != 0
):
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
if residual is None:
dspark_aux_hidden_states.append(hidden_states)
else:
+2 -1
View File
@@ -72,6 +72,7 @@ from sglang.srt.layers.communicator import (
LayerScatterModes,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
)
from sglang.srt.layers.communicator_dsa_cp import (
DSACPLayerCommunicator,
@@ -2688,7 +2689,7 @@ class DeepseekV2DecoderLayer(nn.Module):
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
next_full_attention_layer_id: Optional[int] = None,
) -> torch.Tensor:
hidden_states_orig = hidden_states
hidden_states_orig = layer_input_buffer(hidden_states)
hidden_states, residual = (
self.layer_communicator.prepare_attn_and_capture_last_layer_outputs(
hidden_states,
@@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
UnreducedOutput,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1638,7 +1639,7 @@ class Dots3DecoderLayer(nn.Module):
)
if should_allreduce_fusion:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
if not should_allreduce_fusion:
hidden_states, residual = self.layer_communicator.postprocess_layer(
+2 -2
View File
@@ -41,8 +41,8 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
@@ -1109,7 +1109,7 @@ class Glm4MoeModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(hidden_states + residual)
layer = self.layers[i]
hidden_states, residual = layer(
+2 -2
View File
@@ -35,9 +35,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
get_attn_tp_context,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
@@ -804,7 +804,7 @@ class Glm4MoeLiteModel(nn.Module):
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
if i in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(hidden_states + residual)
layer = self.layers[i]
hidden_states, residual = layer(
+6 -4
View File
@@ -28,9 +28,11 @@ from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
UnreducedOutput,
enable_moe_dense_fully_dp,
get_attn_tp_context,
layer_input_buffer,
reduce_output,
)
from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator
from sglang.srt.layers.layernorm import RMSNorm
@@ -923,7 +925,7 @@ class Glm5NextDecoderLayer(nn.Module):
gemm_output_zero_allocator: BumpAllocator = None,
prev_topk_indices: Optional[torch.Tensor] = None,
):
hidden_states_orig = hidden_states
hidden_states_orig = layer_input_buffer(hidden_states)
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states,
@@ -987,7 +989,7 @@ class Glm5NextDecoderLayer(nn.Module):
)
if should_allreduce_fusion:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
if not should_allreduce_fusion:
hidden_states, residual = self.layer_communicator.postprocess_layer(
@@ -1182,7 +1184,7 @@ class Glm5NextModel(nn.Module):
)
with ctx:
if i in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_state = self._prepare_aux_hidden_state(
hidden_states, residual
)
+2 -2
View File
@@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -725,7 +725,7 @@ class GptOssModel(nn.Module):
positions, hidden_states, forward_batch, residual
)
if i + 1 in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual
if residual is not None
+2 -2
View File
@@ -22,7 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -592,7 +592,7 @@ class LagunaModel(nn.Module):
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
)
+2 -1
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
UnreducedOutput,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1417,7 +1418,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
)
if should_allreduce_fusion:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
+4 -10
View File
@@ -25,15 +25,13 @@ from torch import nn
from sglang.srt.compilation.compilation_config import register_split_op
from sglang.srt.configs import NemotronHConfig
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.layers.activation import ReLU2
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import UnreducedOutput, reduce_output
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
@@ -429,7 +427,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
):
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
@@ -533,7 +531,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
hidden_states, forward_batch, skip_reduce
)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
return hidden_states, residual
@@ -810,11 +808,7 @@ class NemotronHModel(nn.Module):
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
if residual is not None and getattr(
hidden_states, "_sglang_needs_allreduce_fusion", False
):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states._sglang_needs_allreduce_fusion = False
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
self._capture_hidden_states(hidden_states, residual, i)
)
+4 -4
View File
@@ -46,7 +46,8 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
UnreducedOutput,
reduce_output,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -312,8 +313,7 @@ def _finish_mlp_output(hidden_states, *, expect_deferred: bool):
if not expect_deferred:
if isinstance(hidden_states, MoeFinalizeHandoff):
raise RuntimeError("unexpected deferred-finalize handoff")
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states
return UnreducedOutput(hidden_states)
if not isinstance(hidden_states, MoeFinalizeHandoff):
raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff")
@@ -1938,7 +1938,7 @@ class Qwen3_5ForCausalLM(nn.Module):
and layer_idx < 3
):
sep = self.hidden_size * layer_idx
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
hidden_states.add_(
input_deepstack_embeds[:, sep : sep + self.hidden_size]
)
+3 -3
View File
@@ -24,7 +24,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import complete_deferred_allreduce
from sglang.srt.layers.communicator import reduce_output
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import get_layer_id
@@ -107,7 +107,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
):
layer_idx += self.start_layer
if layer_idx in self.layers_to_capture:
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
aux_hidden_states.append(
hidden_states + residual if residual is not None else hidden_states
)
@@ -140,7 +140,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
hidden_states = complete_deferred_allreduce(hidden_states)
hidden_states = reduce_output(hidden_states)
hidden_states.add_(
input_deepstack_embeds[:, sep : sep + self.hidden_size]
)
+2 -1
View File
@@ -14,6 +14,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
UnreducedOutput,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -641,7 +642,7 @@ class Step3p5DecoderLayer(nn.Module):
fuse_mlp_allreduce = False
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
hidden_states = UnreducedOutput(hidden_states)
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch