[ROCm] feat: enable aiter allreduce fusion for GLM models (#39790)

Co-authored-by: xiaobochen-amd <xiaobo.chen@amd.com>
This commit is contained in:
RuibinCheung
2026-09-23 01:24:08 -07:00
committed by GitHub
co-authored by xiaobochen-amd
parent d4dcce12d4
commit 48c3854620
5 changed files with 134 additions and 13 deletions
@@ -656,6 +656,28 @@ class ExecComm(msgspec.Struct):
enable_aiter_allreduce_fusion: A[
bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True)
] = False
disable_aiter_allreduce_fusion_in_prefill: A[
bool,
Arg(
help=(
"Disable Aiter AllReduce Fusion for prefill batches "
"(EXTEND / MIXED / SPLIT_PREFILL) while keeping it for decode. "
"Only meaningful with --enable-aiter-allreduce-fusion."
),
resolvable=True,
),
] = False
disable_aiter_allreduce_fusion_in_decode: A[
bool,
Arg(
help=(
"Disable Aiter AllReduce Fusion for decode batches (DECODE / "
"TARGET_VERIFY / draft-extend / IDLE) while keeping it for prefill. "
"Only meaningful with --enable-aiter-allreduce-fusion."
),
resolvable=True,
),
] = False
class ExecMoe(msgspec.Struct):
+18 -4
View File
@@ -137,7 +137,6 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) -
def handle_model_specific_adjustments(server_args: Any):
cfg = resolving_view(server_args)
from sglang.srt.configs.model_config import (
get_mimo_v2_fused_qkv_expected_tp_size,
@@ -407,7 +406,21 @@ def handle_model_specific_adjustments(server_args: Any):
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
# TODO (Hubert): Put this back later
# server_args.enable_aiter_allreduce_fusion = True
logger.info("Enable Aiter AllReduce Fusion for DeepseekV3ForCausalLM")
if model_arch == "GlmMoeDsaForCausalLM":
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
enable_aiter_allreduce_fusion=True,
)
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
disable_aiter_allreduce_fusion_in_prefill=True,
)
logger.info(
"Enable Aiter AllReduce Fusion on decode phase for GlmMoeDsaForCausalLM"
)
# The fp4-checkpoint draft spec-MoE resolution moved to the
# resolution pipeline (arg_groups/overrides.py:
@@ -500,7 +513,9 @@ def handle_model_specific_adjustments(server_args: Any):
):
# TODO (Hubert): Put this back later
# server_args.enable_aiter_allreduce_fusion = True
logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
# logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
pass
quantization_config = getattr(hf_config, "quantization_config", None)
is_mxfp4_quant_format = (
quantization_config is not None
@@ -950,7 +965,6 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
def handle_language_model_only(server_args: Any):
cfg = resolving_view(server_args)
if not cfg.language_model_only:
return
+16 -5
View File
@@ -205,7 +205,18 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
)
def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
comm = get_exec().comm
if not comm.enable_aiter_allreduce_fusion:
return False
if forward_mode.is_extend_or_draft_extend_or_mixed():
return not comm.disable_aiter_allreduce_fusion_in_prefill
return not comm.disable_aiter_allreduce_fusion_in_decode
def apply_aiter_all_reduce_fusion(
input_tensor: torch.Tensor, forward_batch: ForwardBatch
):
n = input_tensor.shape[-1]
total_bytes = input_tensor.numel() * input_tensor.element_size()
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
@@ -216,7 +227,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
and total_bytes <= 8 * 1024 * 8192
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and get_exec().comm.enable_aiter_allreduce_fusion
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
@@ -700,7 +711,7 @@ class LayerCommunicator:
and hidden_states._sglang_needs_allreduce_fusion
):
if (
apply_aiter_all_reduce_fusion(hidden_states)
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
quant_result = None
@@ -988,7 +999,7 @@ class LayerCommunicator:
and get_parallel().tp_size != 6
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and get_exec().comm.enable_aiter_allreduce_fusion
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
)
)
and (not self.is_last_layer)
@@ -1298,7 +1309,7 @@ class CommunicateWithAllReduceAndLayerNormFn:
else:
handled = False
if (
apply_aiter_all_reduce_fusion(hidden_states)
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
hidden_states, residual = layernorm.forward_with_allreduce_fusion(