[Fix] [AMD] aiter: Fix garbage decode output for non-sliding-window models on the vectorized_5d KV layout (#43335)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
jiaqiang-dot-liu
2026-10-20 11:31:20 +08:00
committed by GitHub
co-authored by Cursor Xiaoyu Zhang
parent 3c07ff2654
commit 1140a8dfd9
2 changed files with 68 additions and 9 deletions
@@ -540,6 +540,14 @@ class AiterAttnBackend(AttentionBackend):
"SGLANG_USE_AITER_UNIFIED_ATTN"
)
# Decode metadata format. unified_attention and pa_decode_gluon (the
# SHUFFLE 5D decode kernel) both read kv_indices as a 2-D
# (bs, max_num_blocks_per_seq) page table; the remaining decode kernels
# read flat per-token kv_indices addressed through kv_indptr.
self.decode_uses_page_table = (
self.use_triton_unified_attention or self.kv_cache_is_vectorized_5d
)
# When topk == 1 the EAGLE draft chain is linear, so target_verify's
# mask reduces to pure causal and can go through unified_attention
# instead of the legacy triton extend_attention_fwd. Gated on non-MLA
@@ -2052,7 +2060,7 @@ class AiterAttnBackend(AttentionBackend):
kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0)
kv_indptr = kv_indptr[: bs + 1]
if not self.use_triton_unified_attention:
if not self.decode_uses_page_table:
kv_indices = self._get_kv_indices_scratch(
forward_batch.seq_lens_sum, forward_batch.seq_lens.device
)
@@ -2117,7 +2125,7 @@ class AiterAttnBackend(AttentionBackend):
qo_indptr = self.qo_indptr_unified_decode[: bs + 1]
else:
if self.use_triton_unified_attention and not self.use_mla:
if self.decode_uses_page_table and not self.use_mla:
bs = spec_info.kv_indptr.shape[0] - 1
kv_indices, swa_page_table = (
self._build_unified_page_table_from_spec(spec_info, bs)
@@ -2817,9 +2825,9 @@ class AiterAttnBackend(AttentionBackend):
else:
self.cuda_graph_kv_indices = kv_indices_buf
if self.use_triton_unified_attention:
# Keep a distinct page-table buffer for unified attention. Sharing
# cuda_graph_kv_indices with non-unified token indices makes
if self.decode_uses_page_table:
# Keep a distinct page-table buffer for page-table decode. Sharing
# cuda_graph_kv_indices with flat per-token indices makes
# page-table width ambiguous after the token buffer is expanded.
max_num_blocks_per_seq = (
self.max_context_len + self.page_size - 1
@@ -2964,14 +2972,12 @@ class AiterAttnBackend(AttentionBackend):
kv_last_page_len = None
max_q_len = None
if spec_info is None or (
self.use_triton_unified_attention and not self.use_mla
):
if spec_info is None or (self.decode_uses_page_table and not self.use_mla):
max_num_blocks_per_seq = (
self.max_context_len + self.page_size - 1
) // self.page_size
if not self.use_triton_unified_attention:
if not self.decode_uses_page_table:
kv_indptr = self.kv_indptr
kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0)
kv_indptr = kv_indptr[: bs + 1]
@@ -0,0 +1,53 @@
"""GSM8K on a dense, non-sliding-window model with the SHUFFLE 5D KV layout.
The 5D layout decodes through ``pa_decode_gluon``, which reads ``kv_indices`` as a
2-D page table. Sliding-window models always get that table because they force
unified attention; a dense model at ``page_size > 1`` only gets it when the decode
metadata follows the KV layout, so this is the configuration that guards it.
"""
import contextlib
import unittest
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_amd_ci(est_time=210, suite="stage-b-test-1-gpu-small-amd-mi35x")
class TestAiterVectorized5DDenseDecode(CustomTestCase, GSM8KMixin):
gsm8k_accuracy_thres = 0.7
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls._env_stack = contextlib.ExitStack()
cls._env_stack.enter_context(envs.SGLANG_USE_AITER.override(True))
cls._env_stack.enter_context(
envs.SGLANG_AITER_KV_CACHE_LAYOUT.override("vectorized_5d")
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--attention-backend", "aiter", "--page-size", "64"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
cls._env_stack.close()
if __name__ == "__main__":
unittest.main()