[Refactor] Let LayerCommunicator own the FFN exit in Qwen3-MoE, DeepSeek-V2 and GLM4-MoE (#40870)
This commit is contained in:
@@ -925,6 +925,11 @@ class LayerCommunicator:
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
)
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
result as a context manager around the FFN call, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch)
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
return False
|
||||
@@ -1039,6 +1044,49 @@ def scatter_mode_layouts(
|
||||
}
|
||||
|
||||
|
||||
class FfnExit:
|
||||
"""One FFN's reduction decision. Inside the ``with`` block it is published as
|
||||
``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``."""
|
||||
|
||||
__slots__ = (
|
||||
"communicator",
|
||||
"forward_batch",
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"_scope",
|
||||
)
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
self.communicator = communicator
|
||||
self.forward_batch = forward_batch
|
||||
self.fuse_mlp_allreduce = (
|
||||
communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
)
|
||||
self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch)
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
)
|
||||
|
||||
def __enter__(self) -> "FfnExit":
|
||||
self._scope.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input norm, or postprocess."""
|
||||
if self.fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states, residual
|
||||
return self.communicator.postprocess_layer(
|
||||
hidden_states, residual, self.forward_batch
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
|
||||
@@ -2692,17 +2692,6 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
if isinstance(self.mlp, DeepseekV2MLP):
|
||||
gemm_output_zero_allocator = None
|
||||
|
||||
@@ -2717,24 +2706,13 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
else:
|
||||
_mlp_ctx = nullcontext()
|
||||
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
gemm_output_zero_allocator,
|
||||
)
|
||||
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
|
||||
if not fuse_mlp_allreduce:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
gemm_output_zero_allocator,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual, topk_indices
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_nextn import DeepseekV3ForCausalLMNextN
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM
|
||||
from sglang.srt.models.utils import WeightsMapper, apply_qk_norm
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
cpu_has_amx_support,
|
||||
@@ -964,29 +964,9 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ from sglang.srt.models.utils import (
|
||||
create_fused_set_kv_buffer_arg,
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import (
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
@@ -907,29 +907,9 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
Reference in New Issue
Block a user