[Refactor] Let LayerCommunicator own the FFN exit in Qwen3-MoE, DeepSeek-V2 and GLM4-MoE (#40870)

This commit is contained in:
Cheng Wan
2026-09-23 14:42:58 -07:00
committed by GitHub
parent 7dd9640752
commit e1048a215c
5 changed files with 190 additions and 74 deletions
+48
View File
@@ -925,6 +925,11 @@ class LayerCommunicator:
allow_reduce_scatter=self.allow_reduce_scatter,
)
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call, then call ``finish``."""
return FfnExit(self, forward_batch)
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
if not self.allow_reduce_scatter:
return False
@@ -1039,6 +1044,49 @@ def scatter_mode_layouts(
}
class FfnExit:
"""One FFN's reduction decision. Inside the ``with`` block it is published as
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``."""
__slots__ = (
"communicator",
"forward_batch",
"fuse_mlp_allreduce",
"mlp_reduce_scatter",
"_scope",
)
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
self.communicator = communicator
self.forward_batch = forward_batch
self.fuse_mlp_allreduce = (
communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
)
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
self._scope = get_forward().scoped(
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
mlp_reduce_scatter=self.mlp_reduce_scatter,
)
def __enter__(self) -> "FfnExit":
self._scope.__enter__()
return self
def __exit__(self, *exc_info):
return self._scope.__exit__(*exc_info)
def finish(
self, hidden_states: torch.Tensor, residual: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Leave the reduction to the next layer's input norm, or postprocess."""
if self.fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states, residual
return self.communicator.postprocess_layer(
hidden_states, residual, self.forward_batch
)
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
+6 -28
View File
@@ -2692,17 +2692,6 @@ class DeepseekV2DecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
if isinstance(self.mlp, DeepseekV2MLP):
gemm_output_zero_allocator = None
@@ -2717,24 +2706,13 @@ class DeepseekV2DecoderLayer(nn.Module):
else:
_mlp_ctx = nullcontext()
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
gemm_output_zero_allocator,
)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
if not fuse_mlp_allreduce:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
hidden_states = self.mlp(
hidden_states,
forward_batch,
gemm_output_zero_allocator,
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual, topk_indices
+3 -23
View File
@@ -83,7 +83,7 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_nextn import DeepseekV3ForCausalLMNextN
from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM
from sglang.srt.models.utils import WeightsMapper, apply_qk_norm
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import (
add_prefix,
cpu_has_amx_support,
@@ -964,29 +964,9 @@ class Glm4MoeDecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual
+3 -23
View File
@@ -63,7 +63,7 @@ from sglang.srt.models.utils import (
create_fused_set_kv_buffer_arg,
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import (
LazyValue,
add_prefix,
@@ -907,29 +907,9 @@ class Qwen3MoeDecoderLayer(nn.Module):
hidden_states, residual, forward_batch
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
# For DP with padding, reduce scatter can be used instead of all-reduce.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
return hidden_states, residual