diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index b9a7f224f4..443b5c090d 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -717,6 +717,8 @@ class LayerCommunicator: force_layernorm_before_dp_gather: bool = False, enable_fused_ar_quant: bool = False, fused_ar_quant_keep_bf16: bool = False, + # False for a layer whose FFN always completes its own reduction. + allow_deferred_ffn_reduction: bool = True, _is_sp_variant: bool = False, ): self.layer_scatter_modes = layer_scatter_modes @@ -728,6 +730,7 @@ class LayerCommunicator: self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather self.enable_fused_ar_quant = enable_fused_ar_quant self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 + self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction self._context = CommunicateContext.init_new() self._context.force_layernorm_before_dp_gather = ( @@ -772,6 +775,7 @@ class LayerCommunicator: force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16, + allow_deferred_ffn_reduction=allow_deferred_ffn_reduction, _is_sp_variant=True, ) @@ -1172,6 +1176,18 @@ class LayerCommunicator: mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( forward_batch, dp_step ) + complete_now = partial( + self._complete_ffn_output_now, + forward_batch=forward_batch, + dp_step=dp_step, + ) + if not self.allow_deferred_ffn_reduction: + return FfnCompletion( + defer_moe_finalize=False, + fuse_mlp_allreduce=False, + mlp_reduce_scatter=mlp_reduce_scatter, + complete=complete_now, + ) defer_moe_finalize = self.should_defer_moe_finalize(forward_batch) # Deferring implies fusing: a handoff skips the post-experts all-reduce. fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( @@ -1206,11 +1222,7 @@ class LayerCommunicator: ), ) else: - complete = partial( - self._complete_ffn_output_now, - forward_batch=forward_batch, - dp_step=dp_step, - ) + complete = complete_now return FfnCompletion( defer_moe_finalize=defer_moe_finalize, fuse_mlp_allreduce=fuse_mlp_allreduce, diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index ee683c118d..ac762fa0ed 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -14,7 +14,7 @@ from functools import partial -from typing import Callable, Optional +from typing import Optional import torch @@ -26,7 +26,6 @@ from sglang.srt.layers.communicator import ( CommunicateSimpleFn, CommunicateSummableTensorPairFn, LayerCommunicator, - LayerScatterModes, ScatterMode, _mlp_input_norm, ) @@ -85,23 +84,6 @@ def dsa_cp_reduce_scatter_hidden_states(hidden_states: torch.Tensor): class DSACPLayerCommunicator(LayerCommunicator): - def __init__( - self, - layer_scatter_modes: LayerScatterModes, - input_layernorm: torch.nn.Module, - post_attention_layernorm: torch.nn.Module, - # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. - allow_reduce_scatter: bool = False, - qkv_latent_func: Optional[Callable] = None, - ): - super().__init__( - layer_scatter_modes, - input_layernorm, - post_attention_layernorm, - allow_reduce_scatter, - qkv_latent_func, - ) - def _post_init_communicate(self): # SCATTERED in attn tp is different from SCATTERED in global tp when dp_size > 1 if self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED: diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 3c87e70516..9542a8bf50 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2628,6 +2628,7 @@ class DeepseekV2DecoderLayer(nn.Module): input_layernorm: nn.Module, post_attention_layernorm: nn.Module, qkv_latent_func: Optional[Callable], + allow_deferred_ffn_reduction: bool = True, ): """The communicator for this layer's norms; it chooses its boundary steps from them at construction.""" @@ -2648,6 +2649,7 @@ class DeepseekV2DecoderLayer(nn.Module): post_attention_layernorm=post_attention_layernorm, allow_reduce_scatter=True, qkv_latent_func=qkv_latent_func, + allow_deferred_ffn_reduction=allow_deferred_ffn_reduction, ) def _detect_gfx95_quant_format(self) -> str: diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 93bcbfe49d..6bf8215c08 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -37,7 +37,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( DeepseekV2WeightLoaderMixin, ) from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet -from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import BumpAllocator, add_prefix, make_layers _GATED_NORM_LOW_RANK = 16 @@ -443,6 +443,7 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): input_layernorm=attn_prepare_layernorm, post_attention_layernorm=mlp_prepare_layernorm, qkv_latent_func=self.layer_communicator.qkv_latent_func, + allow_deferred_ffn_reduction=False, ) def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: @@ -486,24 +487,16 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): hidden_states, residual, forward_batch ) - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) # Unlike deepseek_v2, no moe_output_buffer_ctx here: non-inplace MoE # runners then allocate their output per forward instead of recycling # the layer-input buffer. The default (inplace) runners are unaffected. - with get_forward().scoped( - fuse_mlp_allreduce=False, - mlp_reduce_scatter=mlp_reduce_scatter, - ): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) if self.post_feedforward_layernorm is not None: hidden_states = self.post_feedforward_layernorm(hidden_states) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 79fedb4f58..d30935644a 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -43,7 +43,7 @@ from sglang.srt.models.qwen3_5 import ( _enable_qwen35_fused_ar_quant, _linear_accepts_fp8_tuple, ) -from sglang.srt.runtime_context import get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_parallel, get_stream from sglang.srt.utils import add_prefix, is_cuda, make_layers from sglang.srt.utils.hf_transformers_utils import get_rope_config @@ -423,20 +423,11 @@ class _InternS2MobiusDecoderMixin: hidden_states, residual = self.layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - # Model-side all-reduce fusion is intentionally disabled for baseline. - with get_forward().scoped( - fuse_mlp_allreduce=False, - mlp_reduce_scatter=mlp_reduce_scatter, - ): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self._forward_mobius_mlp( hidden_states, forward_batch, meta_mlp ) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual @@ -507,6 +498,7 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module): allow_reduce_scatter=True, enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=enable_fused_ar_quant, + allow_deferred_ffn_reduction=False, ) def forward( @@ -640,6 +632,7 @@ class InternS2MobiusAttentionDecoderLayer( allow_reduce_scatter=True, enable_fused_ar_quant=enable_fused_ar_quant, fused_ar_quant_keep_bf16=False, + allow_deferred_ffn_reduction=False, ) self.alt_stream = alt_stream diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index def449acf6..760a72ec01 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -71,7 +71,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import ( LazyValue, add_prefix, @@ -649,6 +649,7 @@ class LLaDA2MoeBlock(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) def _is_layer_sparse(self, config: PretrainedConfig, layer_id: int) -> bool: @@ -681,19 +682,9 @@ class LLaDA2MoeBlock(nn.Module): forward_batch=forward_batch, ) - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - - with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index af4f259e1c..a5f4e1cb9e 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -49,7 +49,7 @@ from sglang.srt.model_executor.forward_batch_info import ( ) from sglang.srt.models.llama import LlamaForCausalLM, LlamaMLP from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import ( add_prefix, fast_topk, @@ -433,6 +433,7 @@ class Llama4DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) def _is_moe_layer(self, layer_id: int) -> bool: @@ -468,17 +469,10 @@ class Llama4DecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - # Fully Connected - with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.feed_forward(hidden_states, forward_batch) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 6c3943260e..4ae075531d 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -127,6 +127,7 @@ class MiMoV2MTPLayer(nn.Module): layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, + allow_deferred_ffn_reduction=False, ) def forward( @@ -151,11 +152,12 @@ class MiMoV2MTPLayer(nn.Module): hidden_states, residual = self.layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) - with get_global_expert_distribution_recorder().disable_this_region(): + with ( + self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, + get_global_expert_distribution_recorder().disable_this_region(), + ): hidden_states = self.mlp(hidden_states) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 45c6792ce6..dac1ca0be7 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -1022,6 +1022,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, + allow_deferred_ffn_reduction=False, ) def forward( @@ -1048,10 +1049,9 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): residual, forward_batch, ) - hidden_states = self.mlp(hidden_states) - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 732084936f..6e082a9d77 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -91,7 +91,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.runtime_context import ( get_exec, - get_forward, get_parallel, get_stream, ) @@ -1059,6 +1058,7 @@ class Qwen2MoeDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) def forward( @@ -1092,17 +1092,9 @@ class Qwen2MoeDecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - - with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index dca81ae256..d9f9ca1dc0 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -381,6 +381,7 @@ class Qwen3DecoderLayer(nn.Module): layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, + allow_deferred_ffn_reduction=False, ) def forward( @@ -421,12 +422,11 @@ class Qwen3DecoderLayer(nn.Module): else None ), ) - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) if _is_npu and get_cmo_stream(): wait_cmo_stream() - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 6c8c744a6b..49e3744514 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -40,7 +40,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import add_prefix, is_cuda, make_layers logger = logging.getLogger(__name__) @@ -302,6 +302,7 @@ class SDARBlock(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) def forward( @@ -330,15 +331,9 @@ class SDARBlock(nn.Module): forward_batch, ) - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 0e7b8f6d76..36e41f1a94 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -382,6 +382,7 @@ class Step3TextDecoderLayer(nn.Module): layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, + allow_deferred_ffn_reduction=False, ) def moe_mlp_forward(self, hidden_states): @@ -415,14 +416,12 @@ class Step3TextDecoderLayer(nn.Module): hidden_states, residual = self.layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) - if self.use_moe: - hidden_states = self.moe_mlp_forward(hidden_states) - else: - hidden_states = self.mlp(hidden_states) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + if self.use_moe: + hidden_states = self.moe_mlp_forward(hidden_states) + else: + hidden_states = self.mlp(hidden_states) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index c0c9eb5b77..f29c623a2b 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1542,6 +1542,7 @@ class XllmDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) def forward( @@ -1568,20 +1569,17 @@ class XllmDecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - use_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - - if isinstance(self.mlp, XllmMLP): - hidden_states = self.mlp( - hidden_states, use_reduce_scatter=use_reduce_scatter - ) - else: - hidden_states = self.mlp(hidden_states, forward_batch, use_reduce_scatter) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: + use_reduce_scatter = ffn_exit.mlp_reduce_scatter + if isinstance(self.mlp, XllmMLP): + hidden_states = self.mlp( + hidden_states, use_reduce_scatter=use_reduce_scatter + ) + else: + hidden_states = self.mlp( + hidden_states, forward_batch, use_reduce_scatter + ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 827307cdaa..05c51d251a 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -33,10 +33,12 @@ def make_communicator( scatters_to_local_tokens=False, group=None, cls=LayerCommunicator, + allow_deferred=True, ): """A communicator whose decisions and postprocess are stubbed, built without the process-wide parallel state.""" communicator = cls.__new__(cls) + communicator.allow_deferred_ffn_reduction = allow_deferred if cls is LayerCommunicator: communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse) else: @@ -137,6 +139,17 @@ class TestFfnExit(CustomTestCase): torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) self.assertIs(residual, self.residual) + def test_a_layer_that_declares_no_deferral_runs_postprocess(self): + communicator = make_communicator( + fuse=True, reduce_scatter=True, allow_deferred=False + ) + seen, (hidden_states, residual) = self.run_exit(communicator) + + self.assertEqual(seen, (False, True)) + communicator._ffn_sum_moves_to_next_layer.assert_not_called() + communicator.postprocess_layer.assert_called_once() + torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) + def test_finish_applies_the_decision_the_ffn_saw(self): """finish() applies the decision published to the FFN, even when the decision methods would answer differently by then.""" @@ -267,6 +280,7 @@ class TestSelectFfnCompletion(CustomTestCase): self, *, fuse=False, is_last_layer=False, scatters=True, sp_variant=None ): communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator.allow_deferred_ffn_reduction = True communicator.is_last_layer = is_last_layer communicator._sp_variant = sp_variant communicator._postprocess_scatters_to_local_tokens = scatters diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 577d0780a4..7b2736d05f 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -46,6 +46,7 @@ class DeferringLayer(nn.Module): super().__init__() self.return_topk = return_topk self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + self.layer_communicator.allow_deferred_ffn_reduction = True self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( lambda batch: defer ) diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index 16ba9358b1..7540cfedee 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -58,9 +58,24 @@ def delegates_to_super(forward): ) +def declares_no_deferral(node): + """Builds its communicator with allow_deferred_ffn_reduction=False: its FFN + completes its own reduction.""" + return any( + keyword.arg == "allow_deferred_ffn_reduction" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value is False + for call in ast.walk(node) + if isinstance(call, ast.Call) + for keyword in call.keywords + ) + + def defers(node): """Asks whether to leave the reduction to the next layer (directly or through ffn_exit), or wraps its output as unreduced itself.""" + if declares_no_deferral(node): + return False return any( any(calls(node, name)) for name in ("ffn_exit", "should_fuse_mlp_allreduce_with_next_layer", UNREDUCED) @@ -106,6 +121,7 @@ class Census: if name not in deferring for _, node in defs if set(base_names(node)) & deferring + and not declares_no_deferral(node) and ( method(node, "forward") is None or delegates_to_super(method(node, "forward"))