From 5bb850d536610595ddb6e90ce71342ef68ec7426 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:16:54 -0700 Subject: [PATCH] [Refactor] Split prepare_attn into a reduction step and per-quant-format residual steps (#41084) --- python/sglang/srt/layers/communicator.py | 283 ++++++++---------- .../unit/layers/test_prepare_attn_steps.py | 149 +++++++++ 2 files changed, 281 insertions(+), 151 deletions(-) create mode 100644 test/registered/unit/layers/test_prepare_attn_steps.py diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 9b42e19e18..aa845ef28f 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -539,6 +539,86 @@ def tp_reduce_scatter( return output, residual +def _update_and_read_residual_plain( + norm, hidden_states, residual, post_residual_addition +): + if residual is None: + return norm(hidden_states), hidden_states + return norm(hidden_states, residual, post_residual_addition) + + +def _update_and_read_residual_aiter_mxfp4( + norm, hidden_states, residual, post_residual_addition +): + # post_residual_addition is not applied on this path. + output, *_, residual_out = fused_rms_mxfp4_quant( + hidden_states, + norm.weight, + norm.variance_epsilon, + None, + None, + None, + residual, + ) + return output, hidden_states if residual is None else residual_out + + +def _update_and_read_residual_aiter_fp8_group( + norm, hidden_states, residual, post_residual_addition +): + """aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the + unquantized bf16 output rides along as a third element, so the DSA indexer + can skip dequantizing. post_residual_addition is not applied on this path.""" + needs_bf16 = get_attn_tp_context().is_dsa + output, unquantized, _, residual_out = fused_rms_fp8_group_quant( + hidden_states, + norm.weight, + norm.variance_epsilon, + inp2=None, + inp2_weight=None, + inp2_epsilon=None, + group_size=128, + dtype_quant=torch.float8_e4m3fn, + res1=residual, + output_unquantized_inp1=needs_bf16, + transpose_scale=False, + ) + if _use_aiter_bpreshuffle_gfx95: + output = materialize_bpreshuffle_fp8_scale_tuple(output) + if needs_bf16: + output = (output[0], output[1], unquantized) + return output, hidden_states if residual is None else residual_out + + +def _update_and_read_residual_aiter_fp8_per_token( + norm, hidden_states, residual, post_residual_addition +): + if residual is None: + output = _fused_rmsnorm_fp8_per_token_quant( + hidden_states, norm.weight.data, norm.variance_epsilon + ) + return output, hidden_states + if post_residual_addition is not None: + residual = residual + post_residual_addition + return _fused_rmsnorm_fp8_per_token_quant( + hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual + ) + + +def _attn_input_update_and_read_residual(quant_format: str): + """Add the previous layer's output to the residual and read the attention + input from it: the input norm, fused with the quantization this format + wants. Without a residual (the first layer, or one already folded into + hidden_states) the input itself becomes the residual.""" + if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format: + return _update_and_read_residual_aiter_mxfp4 + if _use_aiter and _is_gfx95_supported and quant_format == "fp8": + return _update_and_read_residual_aiter_fp8_group + if _use_aiter and quant_format == "fp8_per_token": + return _update_and_read_residual_aiter_fp8_per_token + return _update_and_read_residual_plain + + class LayerCommunicator: def __init__( self, @@ -711,158 +791,18 @@ class LayerCommunicator: ) if hidden_states.shape[0] == 0: residual = hidden_states + elif ( + residual is not None + and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") + and hidden_states._sglang_needs_allreduce_fusion + ): + hidden_states, residual = self._reduce_output_and_update_and_read_residual( + hidden_states, residual, forward_batch + ) else: - if ( - residual is not None - and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") - and hidden_states._sglang_needs_allreduce_fusion - ): - if ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): - quant_result = None - if ( - self.enable_fused_ar_quant - and _use_aiter - and hasattr( - self.input_layernorm, - "forward_with_allreduce_fusion_quant_per_group", - ) - ): - # Try fused AR+RMSNorm+per-group-quant. Internally - # falls back to AR+RMSNorm + separate quant when the - # fully-fused kernel cannot service the shape. - quant_result = self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( - hidden_states, - residual, - use_attn_tp_group=False, - keep_bf16=self.fused_ar_quant_keep_bf16, - ) - if quant_result is not None: - hidden_states, residual = quant_result - else: - hidden_states, residual = ( - self.input_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=False - ) - ) - else: - # Fusion was published but this shape can't use the kernel, - # so run the deferred reduction inline. - hidden_states = deferred_post_experts_all_reduce(hidden_states) - hidden_states, residual = self.input_layernorm( - hidden_states, residual - ) - else: - if residual is None: - residual = hidden_states - - if _use_aiter and _is_gfx95_supported and ("mxfp4" in quant_format): - hidden_states, *_, _ = fused_rms_mxfp4_quant( - hidden_states, - self.input_layernorm.weight, - self.input_layernorm.variance_epsilon, - None, - None, - None, - None, - ) - elif _use_aiter and _is_gfx95_supported and (quant_format == "fp8"): - # aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. - # When DSA is active, also preserve the unquantized bf16 - # output as a 3-tuple (fp8, scale, bf16) so the DSA - # indexer can skip redundant FP8 dequantization. - _dsa_needs_bf16 = get_attn_tp_context().is_dsa - hidden_states, _unq_bf16, _, _res = fused_rms_fp8_group_quant( - hidden_states, - self.input_layernorm.weight, - self.input_layernorm.variance_epsilon, - inp2=None, - inp2_weight=None, - inp2_epsilon=None, - group_size=128, - dtype_quant=torch.float8_e4m3fn, - res1=None, - output_unquantized_inp1=_dsa_needs_bf16, - transpose_scale=False, - ) - if _use_aiter_bpreshuffle_gfx95: - hidden_states = materialize_bpreshuffle_fp8_scale_tuple( - hidden_states - ) - if _dsa_needs_bf16: - hidden_states = ( - hidden_states[0], - hidden_states[1], - _unq_bf16, - ) - - elif _use_aiter and (quant_format == "fp8_per_token"): - hidden_states = _fused_rmsnorm_fp8_per_token_quant( - hidden_states, - self.input_layernorm.weight.data, - self.input_layernorm.variance_epsilon, - ) - - else: - hidden_states = self.input_layernorm(hidden_states) - else: - if _use_aiter and _is_gfx95_supported and ("mxfp4" in quant_format): - hidden_states, *_, residual = fused_rms_mxfp4_quant( - hidden_states, - self.input_layernorm.weight, - self.input_layernorm.variance_epsilon, - None, - None, - None, - residual, - ) - elif _use_aiter and _is_gfx95_supported and (quant_format == "fp8"): - # aiter (ROCm gfx95) fused RMSNorm + FP8 group quant - # with residual addition. When DSA is active, pack - # the unquantized bf16 as a 3-tuple (fp8, scale, bf16). - _dsa_needs_bf16 = get_attn_tp_context().is_dsa - hidden_states, _unq_bf16, _, residual = ( - fused_rms_fp8_group_quant( - hidden_states, - self.input_layernorm.weight, - self.input_layernorm.variance_epsilon, - inp2=None, - inp2_weight=None, - inp2_epsilon=None, - group_size=128, - dtype_quant=torch.float8_e4m3fn, - res1=residual, - output_unquantized_inp1=_dsa_needs_bf16, - transpose_scale=False, - ) - ) - if _use_aiter_bpreshuffle_gfx95: - hidden_states = materialize_bpreshuffle_fp8_scale_tuple( - hidden_states - ) - if _dsa_needs_bf16: - hidden_states = ( - hidden_states[0], - hidden_states[1], - _unq_bf16, - ) - elif _use_aiter and (quant_format == "fp8_per_token"): - if post_residual_addition is not None: - residual = residual + post_residual_addition - hidden_states, residual = _fused_rmsnorm_fp8_per_token_quant( - hidden_states, - self.input_layernorm.weight.data, - self.input_layernorm.variance_epsilon, - residual=residual, - ) - else: - hidden_states, residual = self.input_layernorm( - hidden_states, - residual, - post_residual_addition, - ) + hidden_states, residual = _attn_input_update_and_read_residual( + quant_format + )(self.input_layernorm, hidden_states, residual, post_residual_addition) return self._finish_prepare_attn( hidden_states=hidden_states, @@ -884,6 +824,47 @@ class LayerCommunicator: get_attn_tp_context().set_attn_inputs(attn_inputs) return hidden_states, residual + def _reduce_output_and_update_and_read_residual( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Complete the sum the previous layer left, add it to the residual and + apply the input norm: + fused into one kernel when it takes this batch, else as an all-reduce + followed by the norm. Neither applies ``quant_format`` or + ``post_residual_addition``.""" + if ( + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) + or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) + ) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): + if ( + self.enable_fused_ar_quant + and _use_aiter + and hasattr( + self.input_layernorm, + "forward_with_allreduce_fusion_quant_per_group", + ) + ): + # Falls back to AR+RMSNorm + separate quant internally when the + # fully-fused kernel cannot service the shape. + quant_result = ( + self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( + hidden_states, + residual, + use_attn_tp_group=False, + keep_bf16=self.fused_ar_quant_keep_bf16, + ) + ) + if quant_result is not None: + return quant_result + return self.input_layernorm.forward_with_allreduce_fusion( + hidden_states, residual, use_attn_tp_group=False + ) + hidden_states = deferred_post_experts_all_reduce(hidden_states) + return self.input_layernorm(hidden_states, residual) + def _tp_reduce_scatter( self, hidden_states: torch.Tensor, diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py new file mode 100644 index 0000000000..7f63aefdb4 --- /dev/null +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -0,0 +1,149 @@ +"""prepare_attn completes a sum the previous layer left, then adds the residual +and applies the input norm in the form the attention's quant format wants.""" + +import contextlib +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.srt.layers import communicator as comm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +KERNELS = ( + "fused_rms_mxfp4_quant", + "fused_rms_fp8_group_quant", + "_fused_rmsnorm_fp8_per_token_quant", +) + + +class Norm: + weight = torch.ones(4) + variance_epsilon = 1e-6 + + def __init__(self): + self.calls = [] + + def __call__(self, hidden_states, residual=None, post_residual_addition=None): + self.calls.append("norm") + if residual is None: + return hidden_states * 2 + return hidden_states * 2, hidden_states + residual + + def forward_with_allreduce_fusion(self, hidden_states, residual, **_): + self.calls.append("fused_all_reduce_norm") + return hidden_states * 3, hidden_states + residual + + +def communicator(norm): + c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + c.input_layernorm = norm + c._sp_variant = None + c.qkv_latent_func = None + c._context = None + c.enable_fused_ar_quant = False + c.fused_ar_quant_keep_bf16 = False + c._communicate_simple_fn = lambda hidden_states, **_: hidden_states + return c + + +@contextlib.contextmanager +def platform(*, use_aiter=False, gfx95=False, fusion=False): + kernels = {name: MagicMock(name=name) for name in KERNELS} + kernels["fused_rms_mxfp4_quant"].return_value = ("q", None, None, "r") + kernels["fused_rms_fp8_group_quant"].return_value = (("q", "s"), None, None, "r") + kernels["_fused_rmsnorm_fp8_per_token_quant"].return_value = ("q", "r") + all_reduce = MagicMock(side_effect=lambda h: h * 5) + with contextlib.ExitStack() as stack: + for name, mock in kernels.items(): + stack.enter_context(patch.object(comm, name, mock, create=True)) + stack.enter_context(patch.object(comm, "_use_aiter", use_aiter)) + stack.enter_context(patch.object(comm, "_is_gfx95_supported", gfx95)) + stack.enter_context( + patch.object(comm, "_use_aiter_bpreshuffle_gfx95", False, create=True) + ) + stack.enter_context( + patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False) + ) + stack.enter_context( + patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fusion) + ) + stack.enter_context( + patch.object(comm, "deferred_post_experts_all_reduce", all_reduce) + ) + stack.enter_context( + patch.object( + comm, + "get_attn_tp_context", + return_value=SimpleNamespace(input_scattered=False, is_dsa=False), + ) + ) + yield kernels, all_reduce + + +class TestPrepareAttnSteps(CustomTestCase): + def test_each_quant_format_uses_its_kernel(self): + for quant_format, flags, kernel in ( + ("mxfp4", {"use_aiter": True, "gfx95": True}, "fused_rms_mxfp4_quant"), + ("fp8", {"use_aiter": True, "gfx95": True}, "fused_rms_fp8_group_quant"), + ( + "fp8_per_token", + {"use_aiter": True}, + "_fused_rmsnorm_fp8_per_token_quant", + ), + ("fp8", {"use_aiter": True}, None), # the group kernel needs gfx95 + ("mxfp4", {}, None), + ): + for residual in (None, torch.zeros(2, 4)): + with ( + self.subTest( + quant_format=quant_format, + flags=flags, + residual=residual is not None, + ), + platform(**flags) as (kernels, _), + ): + norm = Norm() + communicator(norm).prepare_attn( + torch.ones(2, 4), residual, None, quant_format=quant_format + ) + for name, mock in kernels.items(): + self.assertEqual(mock.called, name == kernel, name) + self.assertEqual(norm.calls, [] if kernel else ["norm"]) + + def test_missing_residual_takes_the_input_as_residual(self): + for quant_format, flags in ( + ("", {}), + ("mxfp4", {"use_aiter": True, "gfx95": True}), + ("fp8", {"use_aiter": True, "gfx95": True}), + ("fp8_per_token", {"use_aiter": True}), + ): + with self.subTest(quant_format=quant_format), platform(**flags): + hidden_states = torch.ones(2, 4) + _, residual = communicator(Norm()).prepare_attn( + hidden_states, None, None, quant_format=quant_format + ) + self.assertIs(residual, hidden_states) + + def test_a_pending_sum_is_completed_once(self): + for fusion in (False, True): + with ( + self.subTest(fusion=fusion), + platform(fusion=fusion) as (_, all_reduce), + ): + norm = Norm() + hidden_states = torch.ones(2, 4) + hidden_states._sglang_needs_allreduce_fusion = True + communicator(norm).prepare_attn(hidden_states, torch.zeros(2, 4), None) + self.assertEqual(all_reduce.call_count, 0 if fusion else 1) + self.assertEqual( + norm.calls, ["fused_all_reduce_norm"] if fusion else ["norm"] + ) + + +if __name__ == "__main__": + unittest.main()