[ROCm] feat: enable aiter allreduce fusion for GLM models (#39790)
Co-authored-by: xiaobochen-amd <xiaobo.chen@amd.com>
This commit is contained in:
co-authored by
xiaobochen-amd
parent
d4dcce12d4
commit
48c3854620
@@ -656,6 +656,28 @@ class ExecComm(msgspec.Struct):
|
||||
enable_aiter_allreduce_fusion: A[
|
||||
bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True)
|
||||
] = False
|
||||
disable_aiter_allreduce_fusion_in_prefill: A[
|
||||
bool,
|
||||
Arg(
|
||||
help=(
|
||||
"Disable Aiter AllReduce Fusion for prefill batches "
|
||||
"(EXTEND / MIXED / SPLIT_PREFILL) while keeping it for decode. "
|
||||
"Only meaningful with --enable-aiter-allreduce-fusion."
|
||||
),
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
disable_aiter_allreduce_fusion_in_decode: A[
|
||||
bool,
|
||||
Arg(
|
||||
help=(
|
||||
"Disable Aiter AllReduce Fusion for decode batches (DECODE / "
|
||||
"TARGET_VERIFY / draft-extend / IDLE) while keeping it for prefill. "
|
||||
"Only meaningful with --enable-aiter-allreduce-fusion."
|
||||
),
|
||||
resolvable=True,
|
||||
),
|
||||
] = False
|
||||
|
||||
|
||||
class ExecMoe(msgspec.Struct):
|
||||
|
||||
@@ -137,7 +137,6 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) -
|
||||
|
||||
|
||||
def handle_model_specific_adjustments(server_args: Any):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_mimo_v2_fused_qkv_expected_tp_size,
|
||||
@@ -407,7 +406,21 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1:
|
||||
# TODO (Hubert): Put this back later
|
||||
# server_args.enable_aiter_allreduce_fusion = True
|
||||
logger.info("Enable Aiter AllReduce Fusion for DeepseekV3ForCausalLM")
|
||||
|
||||
if model_arch == "GlmMoeDsaForCausalLM":
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
enable_aiter_allreduce_fusion=True,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
disable_aiter_allreduce_fusion_in_prefill=True,
|
||||
)
|
||||
logger.info(
|
||||
"Enable Aiter AllReduce Fusion on decode phase for GlmMoeDsaForCausalLM"
|
||||
)
|
||||
|
||||
# The fp4-checkpoint draft spec-MoE resolution moved to the
|
||||
# resolution pipeline (arg_groups/overrides.py:
|
||||
@@ -500,7 +513,9 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
):
|
||||
# TODO (Hubert): Put this back later
|
||||
# server_args.enable_aiter_allreduce_fusion = True
|
||||
logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
|
||||
# logger.info("Enable Aiter AllReduce Fusion for GptOssForCausalLM")
|
||||
pass
|
||||
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
is_mxfp4_quant_format = (
|
||||
quantization_config is not None
|
||||
@@ -950,7 +965,6 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
|
||||
|
||||
|
||||
def handle_language_model_only(server_args: Any):
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.language_model_only:
|
||||
return
|
||||
|
||||
@@ -205,7 +205,18 @@ def apply_flashinfer_allreduce_fusion(batch_size: int):
|
||||
)
|
||||
|
||||
|
||||
def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
|
||||
def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool:
|
||||
comm = get_exec().comm
|
||||
if not comm.enable_aiter_allreduce_fusion:
|
||||
return False
|
||||
if forward_mode.is_extend_or_draft_extend_or_mixed():
|
||||
return not comm.disable_aiter_allreduce_fusion_in_prefill
|
||||
return not comm.disable_aiter_allreduce_fusion_in_decode
|
||||
|
||||
|
||||
def apply_aiter_all_reduce_fusion(
|
||||
input_tensor: torch.Tensor, forward_batch: ForwardBatch
|
||||
):
|
||||
n = input_tensor.shape[-1]
|
||||
total_bytes = input_tensor.numel() * input_tensor.element_size()
|
||||
# Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary.
|
||||
@@ -216,7 +227,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor):
|
||||
and total_bytes <= 8 * 1024 * 8192
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and get_exec().comm.enable_aiter_allreduce_fusion
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
|
||||
|
||||
@@ -700,7 +711,7 @@ class LayerCommunicator:
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
if (
|
||||
apply_aiter_all_reduce_fusion(hidden_states)
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
quant_result = None
|
||||
@@ -988,7 +999,7 @@ class LayerCommunicator:
|
||||
and get_parallel().tp_size != 6
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and get_exec().comm.enable_aiter_allreduce_fusion
|
||||
and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode)
|
||||
)
|
||||
)
|
||||
and (not self.is_last_layer)
|
||||
@@ -1298,7 +1309,7 @@ class CommunicateWithAllReduceAndLayerNormFn:
|
||||
else:
|
||||
handled = False
|
||||
if (
|
||||
apply_aiter_all_reduce_fusion(hidden_states)
|
||||
apply_aiter_all_reduce_fusion(hidden_states, forward_batch)
|
||||
or apply_flashinfer_allreduce_fusion(hidden_states.shape[0])
|
||||
) and hasattr(layernorm, "forward_with_allreduce_fusion"):
|
||||
hidden_states, residual = layernorm.forward_with_allreduce_fusion(
|
||||
|
||||
@@ -13,6 +13,7 @@ import torch
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import CustomTestCase, publish_build_topology
|
||||
|
||||
@@ -358,8 +359,11 @@ def _fake_self(*, mlp_mode=ScatterMode.TP_ATTN_FULL, is_last_layer=False, tp_siz
|
||||
)
|
||||
|
||||
|
||||
def _fake_forward_batch(batch_size=8):
|
||||
return types.SimpleNamespace(input_ids=types.SimpleNamespace(shape=(batch_size,)))
|
||||
def _fake_forward_batch(batch_size=8, forward_mode=ForwardMode.DECODE):
|
||||
return types.SimpleNamespace(
|
||||
input_ids=types.SimpleNamespace(shape=(batch_size,)),
|
||||
forward_mode=forward_mode,
|
||||
)
|
||||
|
||||
|
||||
class TestAiterAllreduceFusionGate(CustomTestCase):
|
||||
@@ -387,6 +391,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
|
||||
mlp_mode=ScatterMode.TP_ATTN_FULL,
|
||||
is_last_layer=False,
|
||||
tp_size=8,
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
disable_in_prefill=False,
|
||||
disable_in_decode=False,
|
||||
):
|
||||
"""Run the gate with the aiter branch isolated (flashinfer forced off)."""
|
||||
a2a_backend = types.SimpleNamespace(is_none=lambda: a2a_is_none)
|
||||
@@ -425,7 +432,9 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
|
||||
|
||||
stack.enter_context(
|
||||
get_context().override_server_args(
|
||||
enable_aiter_allreduce_fusion=aiter_enabled
|
||||
enable_aiter_allreduce_fusion=aiter_enabled,
|
||||
disable_aiter_allreduce_fusion_in_prefill=disable_in_prefill,
|
||||
disable_aiter_allreduce_fusion_in_decode=disable_in_decode,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -438,7 +447,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
|
||||
mlp_mode=mlp_mode, is_last_layer=is_last_layer, tp_size=tp_size
|
||||
)
|
||||
return LayerCommunicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
fake_self, _fake_forward_batch()
|
||||
fake_self, _fake_forward_batch(forward_mode=forward_mode)
|
||||
)
|
||||
|
||||
def test_dense_tp_fuses(self):
|
||||
@@ -479,6 +488,69 @@ class TestAiterAllreduceFusionGate(CustomTestCase):
|
||||
self._evaluate_gate(dp_attention=False, a2a_is_none=True, tp_size=1)
|
||||
)
|
||||
|
||||
PREFILL_MODES = (ForwardMode.EXTEND, ForwardMode.MIXED, ForwardMode.SPLIT_PREFILL)
|
||||
DECODE_MODES = (
|
||||
ForwardMode.DECODE,
|
||||
ForwardMode.TARGET_VERIFY,
|
||||
ForwardMode.DRAFT_EXTEND_V2,
|
||||
ForwardMode.IDLE,
|
||||
)
|
||||
|
||||
def test_prefill_opt_out_disables_fusion_in_prefill_only(self):
|
||||
for mode in self.PREFILL_MODES:
|
||||
with self.subTest(forward_mode=mode):
|
||||
self.assertFalse(
|
||||
self._evaluate_gate(
|
||||
dp_attention=False,
|
||||
a2a_is_none=True,
|
||||
forward_mode=mode,
|
||||
disable_in_prefill=True,
|
||||
)
|
||||
)
|
||||
for mode in self.DECODE_MODES:
|
||||
with self.subTest(forward_mode=mode):
|
||||
self.assertTrue(
|
||||
self._evaluate_gate(
|
||||
dp_attention=False,
|
||||
a2a_is_none=True,
|
||||
forward_mode=mode,
|
||||
disable_in_prefill=True,
|
||||
)
|
||||
)
|
||||
|
||||
def test_decode_opt_out_disables_fusion_in_decode_only(self):
|
||||
for mode in self.DECODE_MODES:
|
||||
with self.subTest(forward_mode=mode):
|
||||
self.assertFalse(
|
||||
self._evaluate_gate(
|
||||
dp_attention=False,
|
||||
a2a_is_none=True,
|
||||
forward_mode=mode,
|
||||
disable_in_decode=True,
|
||||
)
|
||||
)
|
||||
for mode in self.PREFILL_MODES:
|
||||
with self.subTest(forward_mode=mode):
|
||||
self.assertTrue(
|
||||
self._evaluate_gate(
|
||||
dp_attention=False,
|
||||
a2a_is_none=True,
|
||||
forward_mode=mode,
|
||||
disable_in_decode=True,
|
||||
)
|
||||
)
|
||||
|
||||
def test_no_phase_opt_out_fuses_everywhere(self):
|
||||
for mode in self.PREFILL_MODES + self.DECODE_MODES:
|
||||
with self.subTest(forward_mode=mode):
|
||||
self.assertTrue(
|
||||
self._evaluate_gate(
|
||||
dp_attention=False,
|
||||
a2a_is_none=True,
|
||||
forward_mode=mode,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--residual-accuracy" in sys.argv:
|
||||
|
||||
@@ -107,6 +107,8 @@ class TestModelOverridableWhitelist(CustomTestCase):
|
||||
"fp4_gemm_runner_backend",
|
||||
"disable_custom_all_reduce",
|
||||
"enable_aiter_allreduce_fusion",
|
||||
"disable_aiter_allreduce_fusion_in_prefill",
|
||||
"disable_aiter_allreduce_fusion_in_decode",
|
||||
"enable_symm_mem",
|
||||
"speculative_attention_mode",
|
||||
"speculative_draft_attention_backend",
|
||||
|
||||
Reference in New Issue
Block a user