From 48c3854620dd2e111ffaa357f4f986954bb86cc3 Mon Sep 17 00:00:00 2001 From: RuibinCheung Date: Wed, 23 Sep 2026 16:24:08 +0800 Subject: [PATCH] [ROCm] feat: enable aiter allreduce fusion for GLM models (#39790) Co-authored-by: xiaobochen-amd --- python/sglang/srt/arg_groups/fields/exec_.py | 22 +++++ python/sglang/srt/arg_groups/model_hook.py | 22 ++++- python/sglang/srt/layers/communicator.py | 21 +++-- .../ops/test_aiter_allreduce_fusion_amd.py | 80 ++++++++++++++++++- test/registered/unit/test_model_overrides.py | 2 + 5 files changed, 134 insertions(+), 13 deletions(-) diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 8830c3707b..cb763cbd6e 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -656,6 +656,28 @@ class ExecComm(msgspec.Struct): enable_aiter_allreduce_fusion: A[ bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True) ] = False + disable_aiter_allreduce_fusion_in_prefill: A[ + bool, + Arg( + help=( + "Disable Aiter AllReduce Fusion for prefill batches " + "(EXTEND / MIXED / SPLIT_PREFILL) while keeping it for decode. " + "Only meaningful with --enable-aiter-allreduce-fusion." + ), + resolvable=True, + ), + ] = False + disable_aiter_allreduce_fusion_in_decode: A[ + bool, + Arg( + help=( + "Disable Aiter AllReduce Fusion for decode batches (DECODE / " + "TARGET_VERIFY / draft-extend / IDLE) while keeping it for prefill. " + "Only meaningful with --enable-aiter-allreduce-fusion." + ), + resolvable=True, + ), + ] = False class ExecMoe(msgspec.Struct): diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index c4505d0ad7..3c43f705a1 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -137,7 +137,6 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) - def handle_model_specific_adjustments(server_args: Any): - cfg = resolving_view(server_args) from sglang.srt.configs.model_config import ( get_mimo_v2_fused_qkv_expected_tp_size, @@ -407,7 +406,21 @@ def handle_model_specific_adjustments(server_args: Any): if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1: # TODO (Hubert): Put this back later # server_args.enable_aiter_allreduce_fusion = True - logger.info("Enable Aiter AllReduce Fusion for DeepseekV3ForCausalLM") + + if model_arch == "GlmMoeDsaForCausalLM": + declare_resolution( + server_args, + "_handle_model_specific_adjustments", + enable_aiter_allreduce_fusion=True, + ) + declare_resolution( + server_args, + "_handle_model_specific_adjustments", + disable_aiter_allreduce_fusion_in_prefill=True, + ) + logger.info( + "Enable Aiter AllReduce Fusion on decode phase for GlmMoeDsaForCausalLM" + ) # The fp4-checkpoint draft spec-MoE resolution moved to the # resolution pipeline (arg_groups/overrides.py: @@ -500,7 +513,9 @@ def handle_model_specific_adjustments(server_args: Any): ): # TODO (Hubert): Put this back later # server_args.enable_aiter_allreduce_fusion = True - logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM") + # logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM") + pass + quantization_config = getattr(hf_config, "quantization_config", None) is_mxfp4_quant_format = ( quantization_config is not None @@ -950,7 +965,6 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str): def handle_language_model_only(server_args: Any): - cfg = resolving_view(server_args) if not cfg.language_model_only: return diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 4595c53c86..5a7d1e1c9f 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -205,7 +205,18 @@ def apply_flashinfer_allreduce_fusion(batch_size: int): ) -def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor): +def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool: + comm = get_exec().comm + if not comm.enable_aiter_allreduce_fusion: + return False + if forward_mode.is_extend_or_draft_extend_or_mixed(): + return not comm.disable_aiter_allreduce_fusion_in_prefill + return not comm.disable_aiter_allreduce_fusion_in_decode + + +def apply_aiter_all_reduce_fusion( + input_tensor: torch.Tensor, forward_batch: ForwardBatch +): n = input_tensor.shape[-1] total_bytes = input_tensor.numel() * input_tensor.element_size() # Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary. @@ -216,7 +227,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor): and total_bytes <= 8 * 1024 * 8192 and get_parallel().tp_size != 6 and not is_dp_attention_enabled() - and get_exec().comm.enable_aiter_allreduce_fusion + and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) ) @@ -700,7 +711,7 @@ class LayerCommunicator: and hidden_states._sglang_needs_allreduce_fusion ): if ( - apply_aiter_all_reduce_fusion(hidden_states) + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) ) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): quant_result = None @@ -988,7 +999,7 @@ class LayerCommunicator: and get_parallel().tp_size != 6 and not is_dp_attention_enabled() and get_moe_a2a_backend().is_none() - and get_exec().comm.enable_aiter_allreduce_fusion + and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) ) ) and (not self.is_last_layer) @@ -1298,7 +1309,7 @@ class CommunicateWithAllReduceAndLayerNormFn: else: handled = False if ( - apply_aiter_all_reduce_fusion(hidden_states) + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) ) and hasattr(layernorm, "forward_with_allreduce_fusion"): hidden_states, residual = layernorm.forward_with_allreduce_fusion( diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index 8f3edf6320..9099409446 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -13,6 +13,7 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase, publish_build_topology @@ -358,8 +359,11 @@ def _fake_self(*, mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_siz ) -def _fake_forward_batch(batch_size=8): - return types.SimpleNamespace(input_ids=types.SimpleNamespace(shape=(batch_size,))) +def _fake_forward_batch(batch_size=8, forward_mode=ForwardMode.DECODE): + return types.SimpleNamespace( + input_ids=types.SimpleNamespace(shape=(batch_size,)), + forward_mode=forward_mode, + ) class TestAiterAllreduceFusionGate(CustomTestCase): @@ -387,6 +391,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase): mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_size=8, + forward_mode=ForwardMode.DECODE, + disable_in_prefill=False, + disable_in_decode=False, ): """Run the gate with the aiter branch isolated (flashinfer forced off).""" a2a_backend = types.SimpleNamespace(is_none=lambda: a2a_is_none) @@ -425,7 +432,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase): stack.enter_context( get_context().override_server_args( - enable_aiter_allreduce_fusion=aiter_enabled + enable_aiter_allreduce_fusion=aiter_enabled, + disable_aiter_allreduce_fusion_in_prefill=disable_in_prefill, + disable_aiter_allreduce_fusion_in_decode=disable_in_decode, ) ) @@ -438,7 +447,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase): mlp_mode=mlp_mode, is_last_layer=is_last_layer, tp_size=tp_size ) return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer( - fake_self, _fake_forward_batch() + fake_self, _fake_forward_batch(forward_mode=forward_mode) ) def test_dense_tp_fuses(self): @@ -479,6 +488,69 @@ class TestAiterAllreduceFusionGate(CustomTestCase): self._evaluate_gate(dp_attention=False, a2a_is_none=True, tp_size=1) ) + PREFILL_MODES = (ForwardMode.EXTEND, ForwardMode.MIXED, ForwardMode.SPLIT_PREFILL) + DECODE_MODES = ( + ForwardMode.DECODE, + ForwardMode.TARGET_VERIFY, + ForwardMode.DRAFT_EXTEND_V2, + ForwardMode.IDLE, + ) + + def test_prefill_opt_out_disables_fusion_in_prefill_only(self): + for mode in self.PREFILL_MODES: + with self.subTest(forward_mode=mode): + self.assertFalse( + self._evaluate_gate( + dp_attention=False, + a2a_is_none=True, + forward_mode=mode, + disable_in_prefill=True, + ) + ) + for mode in self.DECODE_MODES: + with self.subTest(forward_mode=mode): + self.assertTrue( + self._evaluate_gate( + dp_attention=False, + a2a_is_none=True, + forward_mode=mode, + disable_in_prefill=True, + ) + ) + + def test_decode_opt_out_disables_fusion_in_decode_only(self): + for mode in self.DECODE_MODES: + with self.subTest(forward_mode=mode): + self.assertFalse( + self._evaluate_gate( + dp_attention=False, + a2a_is_none=True, + forward_mode=mode, + disable_in_decode=True, + ) + ) + for mode in self.PREFILL_MODES: + with self.subTest(forward_mode=mode): + self.assertTrue( + self._evaluate_gate( + dp_attention=False, + a2a_is_none=True, + forward_mode=mode, + disable_in_decode=True, + ) + ) + + def test_no_phase_opt_out_fuses_everywhere(self): + for mode in self.PREFILL_MODES + self.DECODE_MODES: + with self.subTest(forward_mode=mode): + self.assertTrue( + self._evaluate_gate( + dp_attention=False, + a2a_is_none=True, + forward_mode=mode, + ) + ) + if __name__ == "__main__": if "--residual-accuracy" in sys.argv: diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 594096c169..3fc8fba80b 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -107,6 +107,8 @@ class TestModelOverridableWhitelist(CustomTestCase): "fp4_gemm_runner_backend", "disable_custom_all_reduce", "enable_aiter_allreduce_fusion", + "disable_aiter_allreduce_fusion_in_prefill", + "disable_aiter_allreduce_fusion_in_decode", "enable_symm_mem", "speculative_attention_mode", "speculative_draft_attention_backend",