[ROCm] feat: enable aiter allreduce fusion for GLM models (#39790)

Co-authored-by: xiaobochen-amd <xiaobo.chen@amd.com>
This commit is contained in:
RuibinCheung
2026-09-23 01:24:08 -07:00
committed by GitHub
co-authored by xiaobochen-amd
parent d4dcce12d4
commit 48c3854620
5 changed files with 134 additions and 13 deletions
@@ -656,6 +656,28 @@ class ExecComm(msgspec.Struct):
enable_aiter_allreduce_fusion: A[
bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True)
] = False
disable_aiter_allreduce_fusion_in_prefill: A[
bool,
Arg(
help=(
"Disable Aiter AllReduce Fusion for prefill batches "
"(EXTEND / MIXED / SPLIT_PREFILL) while keeping it for decode. "
"Only meaningful with --enable-aiter-allreduce-fusion."
),
resolvable=True,
),
] = False
disable_aiter_allreduce_fusion_in_decode: A[
bool,
Arg(
help=(
"Disable Aiter AllReduce Fusion for decode batches (DECODE / "
"TARGET_VERIFY / draft-extend / IDLE) while keeping it for prefill. "
"Only meaningful with --enable-aiter-allreduce-fusion."
),
resolvable=True,
),
] = False
class ExecMoe(msgspec.Struct):
+18 -4
View File
@@ -137,7 +137,6 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) -
def handle_model_specific_adjustments(server_args: Any):
cfg = resolving_view(server_args)
from sglang.srt.configs.model_config import (
get_mimo_v2_fused_qkv_expected_tp_size,
@@ -407,7 +406,21 @@ def handle_model_specific_adjustments(server_args: Any):
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
# TODO (Hubert): Put this back later
# server_args.enable_aiter_allreduce_fusion = True
logger.info("Enable Aiter AllReduce Fusion for DeepseekV3ForCausalLM")
if model_arch == "GlmMoeDsaForCausalLM":
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
enable_aiter_allreduce_fusion=True,
)
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
disable_aiter_allreduce_fusion_in_prefill=True,
)
logger.info(
"Enable Aiter AllReduce Fusion on decode phase for GlmMoeDsaForCausalLM"
)
# The fp4-checkpoint draft spec-MoE resolution moved to the
# resolution pipeline (arg_groups/overrides.py:
@@ -500,7 +513,9 @@ def handle_model_specific_adjustments(server_args: Any):
):
# TODO (Hubert): Put this back later
# server_args.enable_aiter_allreduce_fusion = True
logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
# logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
pass
quantization_config = getattr(hf_config, "quantization_config", None)
is_mxfp4_quant_format = (
quantization_config is not None
@@ -950,7 +965,6 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
def handle_language_model_only(server_args: Any):
cfg = resolving_view(server_args)
if not cfg.language_model_only:
return
+16 -5
View File
@@ -205,7 +205,18 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
)
def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
comm = get_exec().comm
if not comm.enable_aiter_allreduce_fusion:
return False
if forward_mode.is_extend_or_draft_extend_or_mixed():
return not comm.disable_aiter_allreduce_fusion_in_prefill
return not comm.disable_aiter_allreduce_fusion_in_decode
def apply_aiter_all_reduce_fusion(
input_tensor: torch.Tensor, forward_batch: ForwardBatch
):
n = input_tensor.shape[-1]
total_bytes = input_tensor.numel() * input_tensor.element_size()
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
@@ -216,7 +227,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
and total_bytes <= 8 * 1024 * 8192
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and get_exec().comm.enable_aiter_allreduce_fusion
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
@@ -700,7 +711,7 @@ class LayerCommunicator:
and hidden_states._sglang_needs_allreduce_fusion
):
if (
apply_aiter_all_reduce_fusion(hidden_states)
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
quant_result = None
@@ -988,7 +999,7 @@ class LayerCommunicator:
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and get_exec().comm.enable_aiter_allreduce_fusion
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
)
and (not self.is_last_layer)
@@ -1298,7 +1309,7 @@ class CommunicateWithAllReduceAndLayerNormFn:
else:
handled = False
if (
apply_aiter_all_reduce_fusion(hidden_states)
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
hidden_states, residual = layernorm.forward_with_allreduce_fusion(
@@ -13,6 +13,7 @@ import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import CustomTestCase, publish_build_topology
@@ -358,8 +359,11 @@ def _fake_self(*, mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_siz
)
def _fake_forward_batch(batch_size=8):
return types.SimpleNamespace(input_ids=types.SimpleNamespace(shape=(batch_size,)))
def _fake_forward_batch(batch_size=8, forward_mode=ForwardMode.DECODE):
return types.SimpleNamespace(
input_ids=types.SimpleNamespace(shape=(batch_size,)),
forward_mode=forward_mode,
)
class TestAiterAllreduceFusionGate(CustomTestCase):
@@ -387,6 +391,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
mlp_mode=ScatterMode.TP_ATTN_FULL,
is_last_layer=False,
tp_size=8,
forward_mode=ForwardMode.DECODE,
disable_in_prefill=False,
disable_in_decode=False,
):
"""Run the gate with the aiter branch isolated (flashinfer forced off)."""
a2a_backend = types.SimpleNamespace(is_none=lambda: a2a_is_none)
@@ -425,7 +432,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
stack.enter_context(
get_context().override_server_args(
enable_aiter_allreduce_fusion=aiter_enabled
enable_aiter_allreduce_fusion=aiter_enabled,
disable_aiter_allreduce_fusion_in_prefill=disable_in_prefill,
disable_aiter_allreduce_fusion_in_decode=disable_in_decode,
)
)
@@ -438,7 +447,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
mlp_mode=mlp_mode, is_last_layer=is_last_layer, tp_size=tp_size
)
return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer(
fake_self, _fake_forward_batch()
fake_self, _fake_forward_batch(forward_mode=forward_mode)
)
def test_dense_tp_fuses(self):
@@ -479,6 +488,69 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
self._evaluate_gate(dp_attention=False, a2a_is_none=True, tp_size=1)
)
PREFILL_MODES = (ForwardMode.EXTEND, ForwardMode.MIXED, ForwardMode.SPLIT_PREFILL)
DECODE_MODES = (
ForwardMode.DECODE,
ForwardMode.TARGET_VERIFY,
ForwardMode.DRAFT_EXTEND_V2,
ForwardMode.IDLE,
)
def test_prefill_opt_out_disables_fusion_in_prefill_only(self):
for mode in self.PREFILL_MODES:
with self.subTest(forward_mode=mode):
self.assertFalse(
self._evaluate_gate(
dp_attention=False,
a2a_is_none=True,
forward_mode=mode,
disable_in_prefill=True,
)
)
for mode in self.DECODE_MODES:
with self.subTest(forward_mode=mode):
self.assertTrue(
self._evaluate_gate(
dp_attention=False,
a2a_is_none=True,
forward_mode=mode,
disable_in_prefill=True,
)
)
def test_decode_opt_out_disables_fusion_in_decode_only(self):
for mode in self.DECODE_MODES:
with self.subTest(forward_mode=mode):
self.assertFalse(
self._evaluate_gate(
dp_attention=False,
a2a_is_none=True,
forward_mode=mode,
disable_in_decode=True,
)
)
for mode in self.PREFILL_MODES:
with self.subTest(forward_mode=mode):
self.assertTrue(
self._evaluate_gate(
dp_attention=False,
a2a_is_none=True,
forward_mode=mode,
disable_in_decode=True,
)
)
def test_no_phase_opt_out_fuses_everywhere(self):
for mode in self.PREFILL_MODES + self.DECODE_MODES:
with self.subTest(forward_mode=mode):
self.assertTrue(
self._evaluate_gate(
dp_attention=False,
a2a_is_none=True,
forward_mode=mode,
)
)
if __name__ == "__main__":
if "--residual-accuracy" in sys.argv:
@@ -107,6 +107,8 @@ class TestModelOverridableWhitelist(CustomTestCase):
"fp4_gemm_runner_backend",
"disable_custom_all_reduce",
"enable_aiter_allreduce_fusion",
"disable_aiter_allreduce_fusion_in_prefill",
"disable_aiter_allreduce_fusion_in_decode",
"enable_symm_mem",
"speculative_attention_mode",
"speculative_draft_attention_backend",