diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index d4c0430c2a..648f70e571 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1116,6 +1116,17 @@ class FfnExit: ) +def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor: + """Run the all-reduce a layer left for the next layer's input norm.""" + if ( + hasattr(hidden_states, "_sglang_needs_allreduce_fusion") + and hidden_states._sglang_needs_allreduce_fusion + ): + hidden_states = deferred_post_experts_all_reduce(hidden_states) + hidden_states._sglang_needs_allreduce_fusion = False + return hidden_states + + @dataclass class CommunicateContext: process_group_sizes: Dict[ScatterMode, int] diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index fa9615618f..84a7a4f422 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -37,6 +37,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -848,6 +849,7 @@ class BailingMoEModel(nn.Module): ), ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 0d423b55d9..90935af46f 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -18,7 +18,11 @@ from sglang.srt.distributed import ( from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + complete_deferred_allreduce, +) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1008,6 +1012,7 @@ class BailingMoELinearModel(nn.Module): zero_allocator=zero_allocator, ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 188136f91f..c5143e8f5c 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -31,6 +31,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1392,6 +1393,7 @@ class BailingMoELinearModel(nn.Module): dspark_aux_hidden_states.append(hidden_states + residual) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index b8c0177e48..ece4d7c994 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -70,6 +70,7 @@ from sglang.srt.layers.aux_hidden_states import ( from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, get_attn_tp_context, ) @@ -3109,6 +3110,7 @@ class DeepseekV2Model(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) proxy_tensors = { "hidden_states": hidden_states, "residual": residual, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 65d6142441..2b4d18cc25 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1823,6 +1824,7 @@ class Dots3Model(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index e131432a36..6dd084c48c 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -42,6 +42,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1138,6 +1139,7 @@ class Glm4MoeModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 6858116af2..fb01414ad5 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -38,6 +38,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, get_attn_tp_context, ) @@ -837,6 +838,7 @@ class Glm4MoeLiteModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 1c901e997f..90428a02e2 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -28,6 +28,7 @@ from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, get_attn_tp_context, ) @@ -1215,6 +1216,7 @@ class Glm5NextModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) if self.config.mhc: return PPProxyTensors({"hidden_states": hidden_states}) return PPProxyTensors( diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 304235b5a3..01e87aadd8 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -32,7 +32,11 @@ from sglang.srt.distributed import ( ) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + complete_deferred_allreduce, +) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -733,6 +737,7 @@ class GptOssModel(nn.Module): else hidden_states ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 1660a04117..c0c4464f90 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -25,6 +25,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -605,6 +606,7 @@ class LagunaModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 5d6a1d7740..7a4afc3d3d 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1105,6 +1106,7 @@ class MiMoV2Model(nn.Module): hidden_states_before_norm = None if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 6c96c16112..b00735cc21 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, + complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, @@ -1186,6 +1187,7 @@ class MiniMaxM2Model(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index bc4b8dc97a..79a4160d71 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1539,6 +1540,7 @@ class MiniMaxM3Model(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 8f1186b413..44be8131d2 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -34,6 +34,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 +from sglang.srt.layers.communicator import complete_deferred_allreduce from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_dp_attention_enabled, @@ -831,6 +832,7 @@ class NemotronHModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index fe5829b2ce..54de047bbc 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -34,8 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import ( from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.distributed import ( get_pp_indices, - moe_expert_parallel_all_reduce, - moe_tensor_model_parallel_all_reduce, tensor_model_parallel_all_reduce, ) from sglang.srt.environ import envs @@ -47,6 +45,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, + complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -61,7 +60,6 @@ from sglang.srt.layers.linear import ( ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( - can_merge_post_experts_all_reduce, get_moe_a2a_backend, should_skip_post_experts_all_reduce, ) @@ -1231,26 +1229,7 @@ class Qwen2MoeModel(nn.Module): ) if not self.pp_group.is_last_rank: - if ( - hidden_states is not None - and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") - and hidden_states._sglang_needs_allreduce_fusion - ): - # The deferred reduction the next layer would have fused; no - # layer follows on this rank, so run it here. Unconditional -- - # the skip flags that deferred it are what got us into this - # branch -- so it bypasses post_experts_all_reduce()'s guards - # while reusing its merge rule. - if can_merge_post_experts_all_reduce(): - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - else: - if get_parallel().moe_ep_size > 1: - hidden_states = moe_expert_parallel_all_reduce(hidden_states) - if get_parallel().moe_tp_size > 1: - hidden_states = moe_tensor_model_parallel_all_reduce( - hidden_states - ) - hidden_states._sglang_needs_allreduce_fusion = False + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 9fe49041dd..7659a2087b 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -43,7 +43,11 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + complete_deferred_allreduce, +) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1940,6 +1944,7 @@ class Qwen3_5ForCausalLM(nn.Module): # Return intermediate tensors for pipeline parallelism if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index cd7daea022..e4147c0f08 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -24,6 +24,7 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.communicator import complete_deferred_allreduce from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import get_layer_id @@ -150,6 +151,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 3613ebcadb..e34deda704 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -22,6 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1171,6 +1172,7 @@ class SarvamMLAModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 2f59705fb0..f5fee0fc2a 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -16,7 +16,11 @@ from sglang.srt.distributed import ( from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + complete_deferred_allreduce, +) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -487,6 +491,7 @@ class SDARMoeModel(nn.Module): ) if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 2c6cfb6dfb..f0cdc38009 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + complete_deferred_allreduce, +) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -735,6 +739,7 @@ class Step3p5Model(nn.Module): ) # break if not self.pp_group.is_last_rank: + hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index c27a4a9638..dd9228edf9 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -1,9 +1,12 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.communicator import ( + LayerCommunicator, + complete_deferred_allreduce, +) from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -152,5 +155,40 @@ class TestFfnExit(CustomTestCase): self.assertIs(residual, self.residual) +class TestCompleteDeferredAllreduce(CustomTestCase): + def setUp(self): + self.all_reduce = MagicMock(side_effect=lambda hidden_states: hidden_states * 3) + patcher = patch( + "sglang.srt.layers.communicator.deferred_post_experts_all_reduce", + self.all_reduce, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_reduction_left_by_the_last_layer_runs_once(self): + communicator = make_communicator(fuse=True, reduce_scatter=False) + with communicator.ffn_exit(object()) as ffn_exit: + hidden_states = torch.ones(3, 4) + hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) + + hidden_states = complete_deferred_allreduce(hidden_states) + self.all_reduce.assert_called_once() + torch.testing.assert_close(hidden_states, torch.full((3, 4), 3.0)) + self.assertFalse(hidden_states._sglang_needs_allreduce_fusion) + + complete_deferred_allreduce(hidden_states) + self.all_reduce.assert_called_once() + + def test_complete_hidden_states_pass_through(self): + communicator = make_communicator(fuse=False, reduce_scatter=False) + with communicator.ffn_exit(object()) as ffn_exit: + hidden_states = torch.ones(3, 4) + hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) + + self.assertIs(complete_deferred_allreduce(hidden_states), hidden_states) + self.assertIsNone(complete_deferred_allreduce(None)) + self.all_reduce.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py b/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py new file mode 100644 index 0000000000..1e1544766b --- /dev/null +++ b/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py @@ -0,0 +1,174 @@ +"""A decoder layer may leave its FFN all-reduce to the next layer's input norm. +The last layer on a non-last pipeline rank has no next layer there, so every +model forward that sends hidden states produced by such layers must complete +the reduction first. Both sets are derived from the model sources.""" + +import ast +import unittest +from pathlib import Path + +import sglang +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" +MARKER = "_sglang_needs_allreduce_fusion" +COMPLETE = "complete_deferred_allreduce" + + +def base_names(node): + return [ + base.id if isinstance(base, ast.Name) else base.attr + for base in node.bases + if isinstance(base, (ast.Name, ast.Attribute)) + ] + + +def method(node, name): + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == name: + return item + return None + + +def calls(node, name): + for sub in ast.walk(node): + if isinstance(sub, ast.Call): + func = sub.func + called = ( + func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + ) + if called == name: + yield sub + + +def defers_in_own_body(node): + """Asks whether to leave the reduction to the next layer (directly or + through ffn_exit), or sets the marker itself.""" + if any(calls(node, "ffn_exit")) or any( + calls(node, "should_fuse_mlp_allreduce_with_next_layer") + ): + return True + for sub in ast.walk(node): + if isinstance(sub, ast.Assign) and isinstance(sub.value, ast.Constant): + for target in sub.targets: + if ( + isinstance(target, ast.Attribute) + and target.attr == MARKER + and sub.value.value is True + ): + return True + return False + + +def load_classes(): + classes = {} + for path in sorted(MODELS_DIR.rglob("*.py")): + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + classes.setdefault(node.name, []).append((path, tree, node)) + return classes + + +def deferring_layer_classes(classes): + deferring = { + name + for name, defs in classes.items() + if any(defers_in_own_body(node) for _, _, node in defs) + } + # A subclass defers through its base unless it replaces forward without + # delegating to it. + while True: + inherited = set() + for name, defs in classes.items(): + if name in deferring: + continue + for _, _, node in defs: + forward = method(node, "forward") + delegates = forward is None or any( + isinstance(call.func, ast.Attribute) + and isinstance(call.func.value, ast.Call) + and getattr(call.func.value.func, "id", None) == "super" + for call in calls(forward, "forward") + ) + if delegates and set(base_names(node)) & deferring: + inherited.add(name) + if not inherited: + return deferring + deferring |= inherited + + +def uses_deferring_layer(tree, deferring): + """References a deferring layer class other than to subclass it.""" + in_bases = { + id(base) + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) + for base in node.bases + } + for node in ast.walk(tree): + if id(node) in in_bases: + continue + if isinstance(node, ast.Name) and node.id in deferring: + return True + if isinstance(node, ast.Attribute) and node.attr in deferring: + return True + return False + + +def pp_senders(classes): + """forward methods, of models built from deferring layers, that construct + PPProxyTensors -- including ones inherited from a base model class.""" + deferring = deferring_layer_classes(classes) + senders = {} + pending = [] + for defs in classes.values(): + for path, tree, node in defs: + if uses_deferring_layer(tree, deferring): + pending.append((path, node)) + seen = set() + while pending: + path, node = pending.pop() + key = (path, node.name) + if key in seen: + continue + seen.add(key) + forward = method(node, "forward") + if forward is not None and any(calls(forward, "PPProxyTensors")): + senders[f"{path.relative_to(MODELS_DIR)}:{node.name}"] = forward + for base in base_names(node): + for base_path, _, base_node in classes.get(base, ()): + pending.append((base_path, base_node)) + return senders + + +class TestPPSendCompletesDeferredAllreduce(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.classes = load_classes() + cls.senders = pp_senders(cls.classes) + + def test_derivation_finds_the_known_paths(self): + # One model whose own layers defer, one that inherits its forward. + self.assertIn("glm4_moe_lite.py:Glm4MoeLiteModel", self.senders) + self.assertIn("qwen2_moe.py:Qwen2MoeModel", self.senders) + + def test_each_pp_send_completes_the_deferred_reduction_first(self): + missing = [] + for name, forward in sorted(self.senders.items()): + completes = [call.lineno for call in calls(forward, COMPLETE)] + for send in calls(forward, "PPProxyTensors"): + if not any(line < send.lineno for line in completes): + missing.append(f"{name} (line {send.lineno})") + self.assertEqual( + missing, + [], + f"call {COMPLETE}(hidden_states) before sending to the next pipeline rank", + ) + + +if __name__ == "__main__": + unittest.main()