[Refactor] Declare at construction the layers whose FFN completes its own reduction (#41254)
This commit is contained in:
@@ -717,6 +717,8 @@ class LayerCommunicator:
|
||||
force_layernorm_before_dp_gather: bool = False,
|
||||
enable_fused_ar_quant: bool = False,
|
||||
fused_ar_quant_keep_bf16: bool = False,
|
||||
# False for a layer whose FFN always completes its own reduction.
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
_is_sp_variant: bool = False,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
@@ -728,6 +730,7 @@ class LayerCommunicator:
|
||||
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -772,6 +775,7 @@ class LayerCommunicator:
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
_is_sp_variant=True,
|
||||
)
|
||||
|
||||
@@ -1172,6 +1176,18 @@ class LayerCommunicator:
|
||||
mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter(
|
||||
forward_batch, dp_step
|
||||
)
|
||||
complete_now = partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
if not self.allow_deferred_ffn_reduction:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
complete=complete_now,
|
||||
)
|
||||
defer_moe_finalize = self.should_defer_moe_finalize(forward_batch)
|
||||
# Deferring implies fusing: a handoff skips the post-experts all-reduce.
|
||||
fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer(
|
||||
@@ -1206,11 +1222,7 @@ class LayerCommunicator:
|
||||
),
|
||||
)
|
||||
else:
|
||||
complete = partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=dp_step,
|
||||
)
|
||||
complete = complete_now
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=defer_moe_finalize,
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Optional
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -26,7 +26,6 @@ from sglang.srt.layers.communicator import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
_mlp_input_norm,
|
||||
)
|
||||
@@ -85,23 +84,6 @@ def dsa_cp_reduce_scatter_hidden_states(hidden_states: torch.Tensor):
|
||||
|
||||
|
||||
class DSACPLayerCommunicator(LayerCommunicator):
|
||||
def __init__(
|
||||
self,
|
||||
layer_scatter_modes: LayerScatterModes,
|
||||
input_layernorm: torch.nn.Module,
|
||||
post_attention_layernorm: torch.nn.Module,
|
||||
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
|
||||
allow_reduce_scatter: bool = False,
|
||||
qkv_latent_func: Optional[Callable] = None,
|
||||
):
|
||||
super().__init__(
|
||||
layer_scatter_modes,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
allow_reduce_scatter,
|
||||
qkv_latent_func,
|
||||
)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
# SCATTERED in attn tp is different from SCATTERED in global tp when dp_size > 1
|
||||
if self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED:
|
||||
|
||||
@@ -2628,6 +2628,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
input_layernorm: nn.Module,
|
||||
post_attention_layernorm: nn.Module,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
):
|
||||
"""The communicator for this layer's norms; it chooses its boundary
|
||||
steps from them at construction."""
|
||||
@@ -2648,6 +2649,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
)
|
||||
|
||||
def _detect_gfx95_quant_format(self) -> str:
|
||||
|
||||
@@ -37,7 +37,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
DeepseekV2WeightLoaderMixin,
|
||||
)
|
||||
from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import BumpAllocator, add_prefix, make_layers
|
||||
|
||||
_GATED_NORM_LOW_RANK = 16
|
||||
@@ -443,6 +443,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
input_layernorm=attn_prepare_layernorm,
|
||||
post_attention_layernorm=mlp_prepare_layernorm,
|
||||
qkv_latent_func=self.layer_communicator.qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
|
||||
@@ -486,24 +487,16 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
# Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE
|
||||
# runners then allocate their output per forward instead of recycling
|
||||
# the layer-input buffer. The default (inplace) runners are unaffected.
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
if self.post_feedforward_layernorm is not None:
|
||||
hidden_states = self.post_feedforward_layernorm(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ from sglang.srt.models.qwen3_5 import (
|
||||
_enable_qwen35_fused_ar_quant,
|
||||
_linear_accepts_fp8_tuple,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, make_layers
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -423,20 +423,11 @@ class _InternS2MobiusDecoderMixin:
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
# Model-side all-reduce fusion is intentionally disabled for baseline.
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self._forward_mobius_mlp(
|
||||
hidden_states, forward_batch, meta_mlp
|
||||
)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -507,6 +498,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -640,6 +632,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
allow_reduce_scatter=True,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=False,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ from sglang.srt.models.utils import (
|
||||
create_fused_set_kv_buffer_arg,
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import (
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
@@ -649,6 +649,7 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool:
|
||||
@@ -681,19 +682,9 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
)
|
||||
from sglang.srt.models.llama import LlamaForCausalLM, LlamaMLP
|
||||
from sglang.srt.models.utils import apply_qk_norm
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
fast_topk,
|
||||
@@ -433,6 +433,7 @@ class Llama4DecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def _is_moe_layer(self, layer_id: int) -> bool:
|
||||
@@ -468,17 +469,10 @@ class Llama4DecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -127,6 +127,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -151,11 +152,12 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
with (
|
||||
self.layer_communicator.ffn_exit(forward_batch) as ffn_exit,
|
||||
get_global_expert_distribution_recorder().disable_this_region(),
|
||||
):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -1022,6 +1022,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1048,10 +1049,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module):
|
||||
residual,
|
||||
forward_batch,
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -91,7 +91,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
get_stream,
|
||||
)
|
||||
@@ -1059,6 +1058,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1092,17 +1092,9 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
# For DP with padding, reduce scatter can be used instead of all-reduce.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -381,6 +381,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -421,12 +422,11 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
else None
|
||||
),
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
if _is_npu and get_cmo_stream():
|
||||
wait_cmo_stream()
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ from sglang.srt.models.utils import (
|
||||
create_fused_set_kv_buffer_arg,
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, make_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -302,6 +302,7 @@ class SDARBlock(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -330,15 +331,9 @@ class SDARBlock(nn.Module):
|
||||
forward_batch,
|
||||
)
|
||||
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -382,6 +382,7 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
layer_scatter_modes=self.layer_scatter_modes,
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def moe_mlp_forward(self, hidden_states):
|
||||
@@ -415,14 +416,12 @@ class Step3TextDecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
if self.use_moe:
|
||||
hidden_states = self.moe_mlp_forward(hidden_states)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -1542,6 +1542,7 @@ class XllmDecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1568,20 +1569,17 @@ class XllmDecoderLayer(nn.Module):
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
use_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states, use_reduce_scatter=use_reduce_scatter
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch, use_reduce_scatter)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
use_reduce_scatter = ffn_exit.mlp_reduce_scatter
|
||||
if isinstance(self.mlp, XllmMLP):
|
||||
hidden_states = self.mlp(
|
||||
hidden_states, use_reduce_scatter=use_reduce_scatter
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states, forward_batch, use_reduce_scatter
|
||||
)
|
||||
hidden_states, residual = ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
Reference in New Issue
Block a user