From 99c9d65b32c25e760abb79e5d2c260838973dce0 Mon Sep 17 00:00:00 2001 From: chien-an-chen Date: Thu, 1 Oct 2026 13:47:55 +0800 Subject: [PATCH] [AMD] AITER MLA DCP decode ASM path (opt-in) (#40750) --- python/sglang/srt/environ.py | 4 + .../srt/layers/attention/aiter_backend.py | 146 +++++++++++++++--- .../attention/test_aiter_gluon_h12_fp8.py | 76 +++++++++ 3 files changed, 208 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 3b2434e614..207940e525 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -944,6 +944,10 @@ class Envs: # import and Triton cga_layout prerequisites hold. Set to 0 to force the # zero-pad mla_decode_fwd fallback (benchmarking / emergency disable). SGLANG_AITER_MLA_GLUON = EnvBool(True) + # Select the AITER MLA kernel for DCP decode. "gluon" preserves the native + # low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and + # runs mla_decode_fwd with persistent metadata and return_lse=True. + SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon") # DSV4 Aiter flags SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 349d1cf5ab..8c9e278a94 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -325,6 +325,19 @@ class AiterAttnBackend(AttentionBackend): self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA self.dcp_world_size = get_parallel().attn_dcp_size + self.mla_dcp_decode_backend = ( + envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower() + ) + if self.mla_dcp_decode_backend not in ("gluon", "asm"): + raise ValueError( + "SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', " + f"got {self.mla_dcp_decode_backend!r}" + ) + self.use_mla_dcp_asm = ( + self.use_mla + and self.dcp_world_size > 1 + and self.mla_dcp_decode_backend == "asm" + ) # Get v_head_dim based on model type if self.use_mla: @@ -567,9 +580,10 @@ class AiterAttnBackend(AttentionBackend): intra_batch_mode = False log_mla_gluon_capability(logger) - self.max_split_per_batch = 32 if _use_mla_ps_kernel else None + use_any_mla_persist = _use_mla_ps_kernel or self.use_mla_dcp_asm + self.max_split_per_batch = 32 if use_any_mla_persist else None - if self.num_draft_tokens is None and _use_mla_ps_kernel: + if self.num_draft_tokens is None and use_any_mla_persist: self.max_split_per_batch = 64 self.fix_max_split_per_batch = self.max_split_per_batch @@ -624,7 +638,33 @@ class AiterAttnBackend(AttentionBackend): return "auto" return "fp8_e4m3" - def make_mla_decode_meta_data_buffer(self, max_seqlen_qo, batch_size): + def _use_mla_decode_persist_metadata(self) -> bool: + """Persist MLA metadata for non-DCP PS decode, or DCP ASM decode. + + Matches main's ``_use_mla_ps_kernel and dcp_world_size <= 1`` when ASM + is off. DCP ASM needs the same buffers with fast_mode scheduling. + """ + return (_use_mla_ps_kernel and self.dcp_world_size <= 1) or ( + self.use_mla_dcp_asm and self.dcp_world_size > 1 + ) + + def _mla_decode_metadata_modes(self) -> tuple[bool, bool]: + if self.use_mla_dcp_asm and self.dcp_world_size > 1: + # Match the AITER/vLLM DCP path: persistent scheduling with the + # gathered-head tensor folded internally to qh16. + return True, False + return fast_mode, intra_batch_mode + + def make_mla_decode_meta_data_buffer( + self, + max_seqlen_qo, + batch_size, + *, + metadata_fast_mode: Optional[bool] = None, + metadata_intra_batch_mode: Optional[bool] = None, + ): + # Under DCP this is the gathered head count (num_head * dcp_world_size); + # equals num_head_padded when DCP is off. nhead = self.mla_kernel_num_head_padded dtype = self.kv_cache_dtype @@ -636,6 +676,15 @@ class AiterAttnBackend(AttentionBackend): (cu_num + batch_size - 1) // batch_size, self.fix_max_split_per_batch ) + metadata_fast_mode = ( + fast_mode if metadata_fast_mode is None else metadata_fast_mode + ) + metadata_intra_batch_mode = ( + intra_batch_mode + if metadata_intra_batch_mode is None + else metadata_intra_batch_mode + ) + ( (work_meta_data_size, work_meta_data_type), (work_indptr_size, work_indptr_type), @@ -650,9 +699,9 @@ class AiterAttnBackend(AttentionBackend): dtype, dtype, is_sparse=False, - fast_mode=fast_mode, + fast_mode=metadata_fast_mode, num_kv_splits=self.max_split_per_batch, - intra_batch_mode=intra_batch_mode, + intra_batch_mode=metadata_intra_batch_mode, ) # aiter implementation @@ -1343,6 +1392,52 @@ class AiterAttnBackend(AttentionBackend): bs = fm.kv_indptr.shape[0] - 1 num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp + if self.use_mla_dcp_asm: + if fm.work_metadata is None: + raise RuntimeError( + "AITER MLA DCP ASM selected without persistent metadata" + ) + + q_mla = q.view(bs, num_heads, layer.qk_head_dim) + q_scale = k_descale + if q_mla.dtype != fp8_dtype: + q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1)) + q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim) + + out = torch.empty( + (bs, num_heads, layer.v_head_dim), + dtype=self.input_dtype, + device=q.device, + ) + _, lse = mla_decode_fwd( + q_mla, + k_buffer.view(-1, 1, 1, layer.qk_head_dim), + out, + fm.qo_indptr, + fm.kv_indptr[: bs + 1], + fm.kv_indices, + fm.kv_last_page_len, + fm.max_q_len or 1, + sm_scale=layer.scaling, + logit_cap=layer.logit_cap, + num_kv_splits=fm.num_kv_splits, + work_meta_data=fm.work_metadata, + work_indptr=fm.work_indptr, + work_info_set=fm.work_info_set, + reduce_indptr=fm.reduce_indptr, + reduce_final_map=fm.reduce_final_map, + reduce_partial_map=fm.reduce_partial_map, + q_scale=q_scale, + kv_scale=k_descale, + intra_batch_mode=False, + return_lse=True, + ) + if lse is None: + raise RuntimeError( + "aiter mla_decode_fwd(return_lse=True) returned no LSE" + ) + return out, lse.view(bs, num_heads) + out, lse = mla_gluon_decode( q=q.view(bs, num_heads, layer.qk_head_dim), k_buffer=k_buffer, @@ -1658,9 +1753,10 @@ class AiterAttnBackend(AttentionBackend): kv_last_page_len = self.kv_last_page_len[:bs] max_q_len = 1 - # DCP decode runs the aiter MLA kernel (builds its own block-table - # metadata in forward_decode), so skip the persist metadata. - if _use_mla_ps_kernel and self.dcp_world_size <= 1: + if self._use_mla_decode_persist_metadata(): + metadata_fast_mode, metadata_intra_batch_mode = ( + self._mla_decode_metadata_modes() + ) ( work_metadata, work_indptr, @@ -1668,7 +1764,12 @@ class AiterAttnBackend(AttentionBackend): reduce_indptr, reduce_final_map, reduce_partial_map, - ) = self.make_mla_decode_meta_data_buffer(max_q_len, bs) + ) = self.make_mla_decode_meta_data_buffer( + max_q_len, + bs, + metadata_fast_mode=metadata_fast_mode, + metadata_intra_batch_mode=metadata_intra_batch_mode, + ) num_kv_splits = self.max_split_per_batch @@ -1683,9 +1784,9 @@ class AiterAttnBackend(AttentionBackend): reduce_final_map, reduce_partial_map, max_q_len, - fast_mode=fast_mode, + fast_mode=metadata_fast_mode, max_split_per_batch=num_kv_splits, - intra_batch_mode=intra_batch_mode, + intra_batch_mode=metadata_intra_batch_mode, ) self.forward_metadata = ForwardMetadata( @@ -2327,11 +2428,14 @@ class AiterAttnBackend(AttentionBackend): ) # if self.use_mla and (_use_mla_ps_kernel or self.kv_cache_dtype == fp8_dtype): - if self.use_mla and _use_mla_ps_kernel: + if self.use_mla and (_use_mla_ps_kernel or self.use_mla_dcp_asm): # for persistent mla_decode_fwd max_seqlen_qo = ( 1 if self.num_draft_tokens is None else self.num_draft_tokens ) + metadata_fast_mode, metadata_intra_batch_mode = ( + (True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode) + ) ( self.work_metadata, @@ -2340,7 +2444,12 @@ class AiterAttnBackend(AttentionBackend): self.reduce_indptr, self.reduce_final_map, self.reduce_partial_map, - ) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, max_bs) + ) = self.make_mla_decode_meta_data_buffer( + max_seqlen_qo, + max_bs, + metadata_fast_mode=metadata_fast_mode, + metadata_intra_batch_mode=metadata_intra_batch_mode, + ) else: self.work_metadata = None @@ -2500,9 +2609,10 @@ class AiterAttnBackend(AttentionBackend): kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs] max_q_len = 1 - # DCP decode builds its own block-table metadata in - # forward_decode, so the persist metadata is unused here. - if _use_mla_ps_kernel and self.dcp_world_size <= 1: + if self._use_mla_decode_persist_metadata(): + metadata_fast_mode, metadata_intra_batch_mode = ( + self._mla_decode_metadata_modes() + ) num_kv_splits = self.max_split_per_batch self.make_mla_meta_data( @@ -2516,9 +2626,9 @@ class AiterAttnBackend(AttentionBackend): self.reduce_final_map, self.reduce_partial_map, max_q_len, - fast_mode=fast_mode, + fast_mode=metadata_fast_mode, max_split_per_batch=num_kv_splits, - intra_batch_mode=intra_batch_mode, + intra_batch_mode=metadata_intra_batch_mode, ) work_metadata = self.work_metadata diff --git a/test/registered/attention/test_aiter_gluon_h12_fp8.py b/test/registered/attention/test_aiter_gluon_h12_fp8.py index aa759cb3b3..3e8a7efe23 100644 --- a/test/registered/attention/test_aiter_gluon_h12_fp8.py +++ b/test/registered/attention/test_aiter_gluon_h12_fp8.py @@ -317,5 +317,81 @@ class TestForwardMlaDecodeDispatch(CustomTestCase): be._mla_decode_fwd_with_head_pad.assert_not_called() +class TestMlaDcpAsmDecode(CustomTestCase): + def _make_backend(self): + from sglang.srt.layers.attention.aiter_backend import AiterAttnBackend + + be = AiterAttnBackend.__new__(AiterAttnBackend) + be.use_mla_dcp_asm = True + be.dcp_world_size = 8 + be.input_dtype = torch.bfloat16 + be.max_split_per_batch = 64 + be.forward_metadata = mock.Mock( + kv_indptr=torch.tensor([0, 2, 4], dtype=torch.int32), + kv_indices=torch.arange(4, dtype=torch.int32), + kv_last_page_len=torch.ones(2, dtype=torch.int32), + qo_indptr=torch.arange(3, dtype=torch.int32), + max_q_len=1, + work_metadata=torch.empty(1, dtype=torch.int32), + work_indptr=torch.empty(1, dtype=torch.int32), + work_info_set=torch.empty(1, dtype=torch.int32), + reduce_indptr=torch.empty(1, dtype=torch.int32), + reduce_final_map=torch.empty(1, dtype=torch.int32), + reduce_partial_map=torch.empty(1, dtype=torch.int32), + num_kv_splits=64, + ) + return be + + def _make_layer(self): + layer = mock.Mock() + layer.tp_q_head_num = 96 + layer.qk_head_dim = 576 + layer.v_head_dim = 512 + layer.scaling = 0.125 + layer.logit_cap = 0.0 + return layer + + @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_decode_fwd") + @mock.patch("sglang.srt.layers.attention.aiter_backend.scaled_fp8_quant") + def test_asm_quantizes_gathered_q_and_returns_lse(self, mock_quant, mock_mla): + from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype + + be = self._make_backend() + layer = self._make_layer() + q = torch.zeros(2, 96, 576, dtype=torch.bfloat16) + q_fp8 = torch.zeros(2, 96 * 576, dtype=fp8_dtype) + q_scale = torch.tensor([0.25], dtype=torch.float32) + lse = torch.arange(2 * 96, dtype=torch.float32).view(2, 96) + mock_quant.return_value = (q_fp8, q_scale) + mock_mla.return_value = (None, lse) + + out, actual_lse = be._forward_decode_dcp( + q, + torch.zeros(4, 576, dtype=fp8_dtype), + layer, + k_descale=1.0, + ) + + self.assertEqual(out.shape, (2, 96, 512)) + self.assertEqual(out.dtype, torch.bfloat16) + self.assertTrue(torch.equal(actual_lse, lse)) + self.assertEqual(mock_mla.call_args.args[0].dtype, fp8_dtype) + self.assertIs(mock_mla.call_args.kwargs["q_scale"], q_scale) + self.assertTrue(mock_mla.call_args.kwargs["return_lse"]) + self.assertFalse(mock_mla.call_args.kwargs["intra_batch_mode"]) + + def test_dcp_asm_uses_fast_persistent_metadata(self): + be = self._make_backend() + self.assertTrue(be._use_mla_decode_persist_metadata()) + self.assertEqual(be._mla_decode_metadata_modes(), (True, False)) + + def test_default_gluon_dcp_skips_persist_like_main(self): + """DCP>1 without ASM must not allocate persist metadata (main default).""" + be = self._make_backend() + be.use_mla_dcp_asm = False + be.dcp_world_size = 8 + self.assertFalse(be._use_mla_decode_persist_metadata()) + + if __name__ == "__main__": unittest.main()