From 1140a8dfd990dcc9532eaf2779bb287a7092ba0f Mon Sep 17 00:00:00 2001 From: jiaqiang-dot-liu Date: Tue, 20 Oct 2026 11:31:20 +0800 Subject: [PATCH] [Fix] [AMD] aiter: Fix garbage decode output for non-sliding-window models on the vectorized_5d KV layout (#43335) Co-authored-by: Cursor Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../srt/layers/attention/aiter_backend.py | 24 +++++---- .../amd/test_aiter_vectorized_5d_decode.py | 53 +++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 test/registered/amd/test_aiter_vectorized_5d_decode.py diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index fddd5272b6..35054b7670 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -540,6 +540,14 @@ class AiterAttnBackend(AttentionBackend): "SGLANG_USE_AITER_UNIFIED_ATTN" ) + # Decode metadata format. unified_attention and pa_decode_gluon (the + # SHUFFLE 5D decode kernel) both read kv_indices as a 2-D + # (bs, max_num_blocks_per_seq) page table; the remaining decode kernels + # read flat per-token kv_indices addressed through kv_indptr. + self.decode_uses_page_table = ( + self.use_triton_unified_attention or self.kv_cache_is_vectorized_5d + ) + # When topk == 1 the EAGLE draft chain is linear, so target_verify's # mask reduces to pure causal and can go through unified_attention # instead of the legacy triton extend_attention_fwd. Gated on non-MLA @@ -2052,7 +2060,7 @@ class AiterAttnBackend(AttentionBackend): kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) kv_indptr = kv_indptr[: bs + 1] - if not self.use_triton_unified_attention: + if not self.decode_uses_page_table: kv_indices = self._get_kv_indices_scratch( forward_batch.seq_lens_sum, forward_batch.seq_lens.device ) @@ -2117,7 +2125,7 @@ class AiterAttnBackend(AttentionBackend): qo_indptr = self.qo_indptr_unified_decode[: bs + 1] else: - if self.use_triton_unified_attention and not self.use_mla: + if self.decode_uses_page_table and not self.use_mla: bs = spec_info.kv_indptr.shape[0] - 1 kv_indices, swa_page_table = ( self._build_unified_page_table_from_spec(spec_info, bs) @@ -2817,9 +2825,9 @@ class AiterAttnBackend(AttentionBackend): else: self.cuda_graph_kv_indices = kv_indices_buf - if self.use_triton_unified_attention: - # Keep a distinct page-table buffer for unified attention. Sharing - # cuda_graph_kv_indices with non-unified token indices makes + if self.decode_uses_page_table: + # Keep a distinct page-table buffer for page-table decode. Sharing + # cuda_graph_kv_indices with flat per-token indices makes # page-table width ambiguous after the token buffer is expanded. max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 @@ -2964,14 +2972,12 @@ class AiterAttnBackend(AttentionBackend): kv_last_page_len = None max_q_len = None - if spec_info is None or ( - self.use_triton_unified_attention and not self.use_mla - ): + if spec_info is None or (self.decode_uses_page_table and not self.use_mla): max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 ) // self.page_size - if not self.use_triton_unified_attention: + if not self.decode_uses_page_table: kv_indptr = self.kv_indptr kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) kv_indptr = kv_indptr[: bs + 1] diff --git a/test/registered/amd/test_aiter_vectorized_5d_decode.py b/test/registered/amd/test_aiter_vectorized_5d_decode.py new file mode 100644 index 0000000000..ccb2682984 --- /dev/null +++ b/test/registered/amd/test_aiter_vectorized_5d_decode.py @@ -0,0 +1,53 @@ +"""GSM8K on a dense, non-sliding-window model with the SHUFFLE 5D KV layout. + +The 5D layout decodes through ``pa_decode_gluon``, which reads ``kv_indices`` as a +2-D page table. Sliding-window models always get that table because they force +unified attention; a dense model at ``page_size > 1`` only gets it when the decode +metadata follows the KV layout, so this is the configuration that guards it. +""" + +import contextlib +import unittest + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_amd_ci(est_time=210, suite="stage-b-test-1-gpu-small-amd-mi35x") + + +class TestAiterVectorized5DDenseDecode(CustomTestCase, GSM8KMixin): + gsm8k_accuracy_thres = 0.7 + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls._env_stack = contextlib.ExitStack() + cls._env_stack.enter_context(envs.SGLANG_USE_AITER.override(True)) + cls._env_stack.enter_context( + envs.SGLANG_AITER_KV_CACHE_LAYOUT.override("vectorized_5d") + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "aiter", "--page-size", "64"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls._env_stack.close() + + +if __name__ == "__main__": + unittest.main()