[Refactor] Declare at construction the layers whose FFN completes its own reduction (#41254)

This commit is contained in:
Cheng Wan
2026-09-26 02:39:58 -07:00
committed by GitHub
parent c081d4aacc
commit 35b369ebba
17 changed files with 108 additions and 124 deletions
+17 -5
View File
@@ -717,6 +717,8 @@ class LayerCommunicator:
force_layernorm_before_dp_gather: bool = False,
enable_fused_ar_quant: bool = False,
fused_ar_quant_keep_bf16: bool = False,
# False for a layer whose FFN always completes its own reduction.
allow_deferred_ffn_reduction: bool = True,
_is_sp_variant: bool = False,
):
self.layer_scatter_modes = layer_scatter_modes
@@ -728,6 +730,7 @@ class LayerCommunicator:
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
self.enable_fused_ar_quant = enable_fused_ar_quant
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
self._context = CommunicateContext.init_new()
self._context.force_layernorm_before_dp_gather = (
@@ -772,6 +775,7 @@ class LayerCommunicator:
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
_is_sp_variant=True,
)
@@ -1172,6 +1176,18 @@ class LayerCommunicator:
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
forward_batch, dp_step
)
complete_now = partial(
self._complete_ffn_output_now,
forward_batch=forward_batch,
dp_step=dp_step,
)
if not self.allow_deferred_ffn_reduction:
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
complete=complete_now,
)
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
@@ -1206,11 +1222,7 @@ class LayerCommunicator:
),
)
else:
complete = partial(
self._complete_ffn_output_now,
forward_batch=forward_batch,
dp_step=dp_step,
)
complete = complete_now
return FfnCompletion(
defer_moe_finalize=defer_moe_finalize,
fuse_mlp_allreduce=fuse_mlp_allreduce,
@@ -14,7 +14,7 @@
from functools import partial
from typing import Callable, Optional
from typing import Optional
import torch
@@ -26,7 +26,6 @@ from sglang.srt.layers.communicator import (
CommunicateSimpleFn,
CommunicateSummableTensorPairFn,
LayerCommunicator,
LayerScatterModes,
ScatterMode,
_mlp_input_norm,
)
@@ -85,23 +84,6 @@ def dsa_cp_reduce_scatter_hidden_states(hidden_states: torch.Tensor):
class DSACPLayerCommunicator(LayerCommunicator):
def __init__(
self,
layer_scatter_modes: LayerScatterModes,
input_layernorm: torch.nn.Module,
post_attention_layernorm: torch.nn.Module,
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
allow_reduce_scatter: bool = False,
qkv_latent_func: Optional[Callable] = None,
):
super().__init__(
layer_scatter_modes,
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
qkv_latent_func,
)
def _post_init_communicate(self):
# SCATTERED in attn tp is different from SCATTERED in global tp when dp_size > 1
if self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED:
+2
View File
@@ -2628,6 +2628,7 @@ class DeepseekV2DecoderLayer(nn.Module):
input_layernorm: nn.Module,
post_attention_layernorm: nn.Module,
qkv_latent_func: Optional[Callable],
allow_deferred_ffn_reduction: bool = True,
):
"""The communicator for this layer's norms; it chooses its boundary
steps from them at construction."""
@@ -2648,6 +2649,7 @@ class DeepseekV2DecoderLayer(nn.Module):
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=True,
qkv_latent_func=qkv_latent_func,
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
)
def _detect_gfx95_quant_format(self) -> str:
+4 -11
View File
@@ -37,7 +37,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
DeepseekV2WeightLoaderMixin,
)
from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import BumpAllocator, add_prefix, make_layers
_GATED_NORM_LOW_RANK = 16
@@ -443,6 +443,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
input_layernorm=attn_prepare_layernorm,
post_attention_layernorm=mlp_prepare_layernorm,
qkv_latent_func=self.layer_communicator.qkv_latent_func,
allow_deferred_ffn_reduction=False,
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
@@ -486,24 +487,16 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
hidden_states, residual, forward_batch
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
# runners then allocate their output per forward instead of recycling
# the layer-input buffer. The default (inplace) runners are unaffected.
with get_forward().scoped(
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if self.post_feedforward_layernorm is not None:
hidden_states = self.post_feedforward_layernorm(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+5 -12
View File
@@ -43,7 +43,7 @@ from sglang.srt.models.qwen3_5 import (
_enable_qwen35_fused_ar_quant,
_linear_accepts_fp8_tuple,
)
from sglang.srt.runtime_context import get_forward, get_parallel, get_stream
from sglang.srt.runtime_context import get_parallel, get_stream
from sglang.srt.utils import add_prefix, is_cuda, make_layers
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -423,20 +423,11 @@ class _InternS2MobiusDecoderMixin:
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
# Model-side all-reduce fusion is intentionally disabled for baseline.
with get_forward().scoped(
fuse_mlp_allreduce=False,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self._forward_mobius_mlp(
hidden_states, forward_batch, meta_mlp
)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
@@ -507,6 +498,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -640,6 +632,7 @@ class InternS2MobiusAttentionDecoderLayer(
allow_reduce_scatter=True,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=False,
allow_deferred_ffn_reduction=False,
)
self.alt_stream = alt_stream
+4 -13
View File
@@ -71,7 +71,7 @@ from sglang.srt.models.utils import (
create_fused_set_kv_buffer_arg,
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import (
LazyValue,
add_prefix,
@@ -649,6 +649,7 @@ class LLaDA2MoeBlock(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
)
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
@@ -681,19 +682,9 @@ class LLaDA2MoeBlock(nn.Module):
forward_batch=forward_batch,
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+4 -10
View File
@@ -49,7 +49,7 @@ from sglang.srt.model_executor.forward_batch_info import (
)
from sglang.srt.models.llama import LlamaForCausalLM, LlamaMLP
from sglang.srt.models.utils import apply_qk_norm
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import (
add_prefix,
fast_topk,
@@ -433,6 +433,7 @@ class Llama4DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
)
def _is_moe_layer(self, layer_id: int) -> bool:
@@ -468,17 +469,10 @@ class Llama4DecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
# Fully Connected
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.feed_forward(hidden_states, forward_batch)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+6 -4
View File
@@ -127,6 +127,7 @@ class MiMoV2MTPLayer(nn.Module):
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -151,11 +152,12 @@ class MiMoV2MTPLayer(nn.Module):
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
with get_global_expert_distribution_recorder().disable_this_region():
with (
self.layer_communicator.ffn_exit(forward_batch) as ffn_exit,
get_global_expert_distribution_recorder().disable_this_region(),
):
hidden_states = self.mlp(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+4 -4
View File
@@ -1022,6 +1022,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -1048,10 +1049,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
residual,
forward_batch,
)
hidden_states = self.mlp(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+3 -11
View File
@@ -91,7 +91,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_parallel,
get_stream,
)
@@ -1059,6 +1058,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -1092,17 +1092,9 @@ class Qwen2MoeDecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+4 -4
View File
@@ -381,6 +381,7 @@ class Qwen3DecoderLayer(nn.Module):
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -421,12 +422,11 @@ class Qwen3DecoderLayer(nn.Module):
else None
),
)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+4 -9
View File
@@ -40,7 +40,7 @@ from sglang.srt.models.utils import (
create_fused_set_kv_buffer_arg,
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, is_cuda, make_layers
logger = logging.getLogger(__name__)
@@ -302,6 +302,7 @@ class SDARBlock(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -330,15 +331,9 @@ class SDARBlock(nn.Module):
forward_batch,
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+7 -8
View File
@@ -382,6 +382,7 @@ class Step3TextDecoderLayer(nn.Module):
layer_scatter_modes=self.layer_scatter_modes,
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_deferred_ffn_reduction=False,
)
def moe_mlp_forward(self, hidden_states):
@@ -415,14 +416,12 @@ class Step3TextDecoderLayer(nn.Module):
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
if self.use_moe:
hidden_states = self.moe_mlp_forward(hidden_states)
else:
hidden_states = self.mlp(hidden_states)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+12 -14
View File
@@ -1542,6 +1542,7 @@ class XllmDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
allow_deferred_ffn_reduction=False,
)
def forward(
@@ -1568,20 +1569,17 @@ class XllmDecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
use_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(
hidden_states, use_reduce_scatter=use_reduce_scatter
)
else:
hidden_states = self.mlp(hidden_states, forward_batch, use_reduce_scatter)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
use_reduce_scatter = ffn_exit.mlp_reduce_scatter
if isinstance(self.mlp, XllmMLP):
hidden_states = self.mlp(
hidden_states, use_reduce_scatter=use_reduce_scatter
)
else:
hidden_states = self.mlp(
hidden_states, forward_batch, use_reduce_scatter
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual