[ROCm] feat: enable aiter allreduce fusion for GLM models (#39790)
Co-authored-by: xiaobochen-amd <xiaobo.chen@amd.com>
This commit is contained in:
co-authored by
xiaobochen-amd
parent
d4dcce12d4
commit
48c3854620
@@ -656,6 +656,28 @@ class ExecComm(msgspec.Struct):
|
||||
enable_aiter_allreduce_fusion: A[
|
||||
bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True)
|
||||
] = False
|
||||
disable_aiter_allreduce_fusion_in_prefill: A[
|
||||
bool,
|
||||
Arg(
|
||||
help=(
|
||||
"Disable Aiter AllReduce Fusion for prefill batches "
|
||||
"(EXTEND / MIXED / SPLIT_PREFILL) while keeping it for decode. "
|
||||
"Only meaningful with --enable-aiter-allreduce-fusion."
|
||||
),
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
disable_aiter_allreduce_fusion_in_decode: A[
|
||||
bool,
|
||||
Arg(
|
||||
help=(
|
||||
"Disable Aiter AllReduce Fusion for decode batches (DECODE / "
|
||||
"TARGET_VERIFY / draft-extend / IDLE) while keeping it for prefill. "
|
||||
"Only meaningful with --enable-aiter-allreduce-fusion."
|
||||
),
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
|
||||
|
||||
class ExecMoe(msgspec.Struct):
|
||||
|
||||
@@ -137,7 +137,6 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) -
|
||||
|
||||
|
||||
def handle_model_specific_adjustments(server_args: Any):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_mimo_v2_fused_qkv_expected_tp_size,
|
||||
@@ -407,7 +406,21 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
|
||||
# TODO (Hubert): Put this back later
|
||||
# server_args.enable_aiter_allreduce_fusion = True
|
||||
logger.info("Enable Aiter AllReduce Fusion for DeepseekV3ForCausalLM")
|
||||
|
||||
if model_arch == "GlmMoeDsaForCausalLM":
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
enable_aiter_allreduce_fusion=True,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
disable_aiter_allreduce_fusion_in_prefill=True,
|
||||
)
|
||||
logger.info(
|
||||
"Enable Aiter AllReduce Fusion on decode phase for GlmMoeDsaForCausalLM"
|
||||
)
|
||||
|
||||
# The fp4-checkpoint draft spec-MoE resolution moved to the
|
||||
# resolution pipeline (arg_groups/overrides.py:
|
||||
@@ -500,7 +513,9 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
):
|
||||
# TODO (Hubert): Put this back later
|
||||
# server_args.enable_aiter_allreduce_fusion = True
|
||||
logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
|
||||
# logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
|
||||
pass
|
||||
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
is_mxfp4_quant_format = (
|
||||
quantization_config is not None
|
||||
@@ -950,7 +965,6 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
|
||||
|
||||
|
||||
def handle_language_model_only(server_args: Any):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.language_model_only:
|
||||
return
|
||||
|
||||
@@ -205,7 +205,18 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
|
||||
)
|
||||
|
||||
|
||||
def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
|
||||
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
|
||||
comm = get_exec().comm
|
||||
if not comm.enable_aiter_allreduce_fusion:
|
||||
return False
|
||||
if forward_mode.is_extend_or_draft_extend_or_mixed():
|
||||
return not comm.disable_aiter_allreduce_fusion_in_prefill
|
||||
return not comm.disable_aiter_allreduce_fusion_in_decode
|
||||
|
||||
|
||||
def apply_aiter_all_reduce_fusion(
|
||||
input_tensor: torch.Tensor, forward_batch: ForwardBatch
|
||||
):
|
||||
n = input_tensor.shape[-1]
|
||||
total_bytes = input_tensor.numel() * input_tensor.element_size()
|
||||
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
|
||||
@@ -216,7 +227,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
|
||||
and total_bytes <= 8 * 1024 * 8192
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and get_exec().comm.enable_aiter_allreduce_fusion
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
|
||||
|
||||
@@ -700,7 +711,7 @@ class LayerCommunicator:
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
if (
|
||||
apply_aiter_all_reduce_fusion(hidden_states)
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
quant_result = None
|
||||
@@ -988,7 +999,7 @@ class LayerCommunicator:
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and get_exec().comm.enable_aiter_allreduce_fusion
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
)
|
||||
and (not self.is_last_layer)
|
||||
@@ -1298,7 +1309,7 @@ class CommunicateWithAllReduceAndLayerNormFn:
|
||||
else:
|
||||
handled = False
|
||||
if (
|
||||
apply_aiter_all_reduce_fusion(hidden_states)
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
|
||||
hidden_states, residual = layernorm.forward_with_allreduce_fusion(
|
||||
|
||||
Reference in New Issue
Block a user