diff --git a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py index f223c5534b..4bb72a6e1e 100644 --- a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py +++ b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py @@ -33,6 +33,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import ( TRTLLMMLAMultiStepDraftBackend, ) from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run +from sglang.srt.mem_cache.layout.paged_view import paged_row_view from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_flashinfer_available @@ -226,7 +227,7 @@ class CuteDslMLABackend(TRTLLMMLABackend): query = query.unsqueeze(1) k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) + kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1) metadata = ( getattr(forward_batch, "decode_trtllm_mla_metadata", None) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 5559e319ed..bc31bbc92e 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -34,6 +34,7 @@ from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.mem_cache.kv_index_translator import KVReadTables +from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode @@ -1305,20 +1306,33 @@ class FlashAttentionBackend(AttentionBackend): head_group_num: int = 1, ) -> tuple[torch.Tensor, torch.Tensor]: key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) - return ( - key_cache.view( + key_cache = paged_kv_view( + key_cache, self.page_size, layer.tp_k_head_num, layer.head_dim + ) + value_cache = paged_kv_view( + value_cache, self.page_size, layer.tp_v_head_num, layer.v_head_dim + ) + if head_group_num != 1: + # Reinterpret each page's heads as `head_group_num` pseudo-pages. + assert key_cache.is_contiguous() and value_cache.is_contiguous(), ( + f"head_group_num={head_group_num} needs a contiguous paged KV " + "view; this one carries the pool's slot stride " + f"(key {key_cache.stride()}, value {value_cache.stride()}), " + "which no reshape of the head dimension can express." + ) + key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num // head_group_num, layer.head_dim, - ), - value_cache.view( + ) + value_cache = value_cache.view( -1, self.page_size, layer.tp_v_head_num // head_group_num, layer.v_head_dim, - ), - ) + ) + return key_cache, value_cache def prepare_paged_mha_query( self, @@ -1695,9 +1709,12 @@ class FlashAttentionBackend(AttentionBackend): q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), # The suffix table stores physical token slots, so expose # the paged cache as page-size-one blocks. - k_cache=key_cache.view(-1, 1, layer.tp_k_head_num, layer.head_dim), - v_cache=value_cache.view( - -1, 1, layer.tp_v_head_num, layer.v_head_dim + k_cache=paged_view( + key_cache.view(-1, layer.tp_k_head_num, layer.head_dim), 1 + ), + v_cache=paged_view( + value_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim), + 1, ), page_table=self.forward_metadata_spec_decode_expand.page_table, cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, @@ -1797,15 +1814,8 @@ class FlashAttentionBackend(AttentionBackend): ) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] - k_rope_cache = k_rope.view( - -1, - self.page_size, - layer.tp_k_head_num, - layer.head_dim - layer.v_head_dim, - ) - c_kv_cache = c_kv.view( - -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim - ) + k_rope_cache = paged_view(k_rope, self.page_size) + c_kv_cache = paged_view(c_kv, self.page_size) if q_rope is not None: q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) q_rope = q_rope.view( @@ -2209,15 +2219,8 @@ class FlashAttentionBackend(AttentionBackend): kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] - k_rope_cache = k_rope.view( - -1, - self.page_size, - layer.tp_k_head_num, - layer.head_dim - layer.v_head_dim, - ) - c_kv_cache = c_kv.view( - -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim - ) + k_rope_cache = paged_view(k_rope, self.page_size) + c_kv_cache = paged_view(c_kv, self.page_size) if q_rope is not None: q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 42a4d38de7..43439a2632 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -19,6 +19,7 @@ from sglang.kernels.ops.attention.utils import ( from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask +from sglang.srt.mem_cache.layout.paged_view import paged_view from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import get_parallel, get_spec @@ -461,7 +462,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape) o, _ = flash_mla_with_kvcache( q=reshape_q_fp8, - k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + k_cache=paged_view(k_cache, PAGE_SIZE), block_table=self.forward_metadata.block_kv_indices[:bs], cache_seqlens=forward_batch.seq_lens.to(torch.int32), head_dim_v=self.kv_lora_rank, @@ -478,7 +479,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): # todo: need check all causal True or False? o, lse = flash_mla_with_kvcache( q=reshape_q, - k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + k_cache=paged_view(k_cache, PAGE_SIZE), block_table=self.forward_metadata.block_kv_indices[:bs], cache_seqlens=forward_batch.seq_lens.to(torch.int32), head_dim_v=self.kv_lora_rank, @@ -552,7 +553,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape) o, _ = flash_mla_with_kvcache( q=reshape_q_fp8, - k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + k_cache=paged_view(k_cache, PAGE_SIZE), block_table=self.forward_metadata.block_kv_indices[:bs], cache_seqlens=cache_seqlens, head_dim_v=self.kv_lora_rank, @@ -566,7 +567,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): else: o, _ = flash_mla_with_kvcache( q=reshape_q, - k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + k_cache=paged_view(k_cache, PAGE_SIZE), block_table=self.forward_metadata.block_kv_indices[:bs], cache_seqlens=cache_seqlens, head_dim_v=self.kv_lora_rank, diff --git a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py index 3338325458..e65a0e9354 100644 --- a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py +++ b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py @@ -49,6 +49,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import ( TRTLLMMLAMultiStepDraftBackend, ) from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run +from sglang.srt.mem_cache.layout.paged_view import paged_row_view from sglang.srt.runtime_context import ( get_parallel, get_resources, @@ -411,7 +412,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend): query = query.unsqueeze(1) k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) + kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1) metadata = ( getattr(forward_batch, "decode_trtllm_mla_metadata", None) or self.forward_decode_metadata diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index c10e6b8b5d..2f723bf7f3 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( KVCacheAttentionAccessKind, ) from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.mem_cache.layout.paged_view import paged_kv_view from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode @@ -1314,11 +1315,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): layer: RadixAttention, head_dim: int, ) -> tuple[torch.Tensor, torch.Tensor]: - k_cache = k_cache.view( - -1, self.page_size, layer.tp_k_head_num, head_dim + k_cache = paged_kv_view( + k_cache, self.page_size, layer.tp_k_head_num, head_dim ).permute(0, 2, 1, 3) - v_cache = v_cache.view( - -1, self.page_size, layer.tp_v_head_num, head_dim + v_cache = paged_kv_view( + v_cache, self.page_size, layer.tp_v_head_num, head_dim ).permute(0, 2, 1, 3) if layer.tp_k_head_num == 1: k_cache = canonicalize_stride(k_cache) @@ -1632,11 +1633,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): k_cache_raw, v_cache_raw, layer, layer.head_dim ) else: - k_cache = k_cache_raw.view( - -1, self.page_size, layer.tp_k_head_num, layer.head_dim + k_cache = paged_kv_view( + k_cache_raw, self.page_size, layer.tp_k_head_num, layer.head_dim ) - v_cache = v_cache_raw.view( - -1, self.page_size, layer.tp_v_head_num, layer.head_dim + v_cache = paged_kv_view( + v_cache_raw, self.page_size, layer.tp_v_head_num, layer.head_dim ) kv_cache = (k_cache, v_cache) diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index bfe20646af..62123f12ea 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -57,6 +57,7 @@ from sglang.srt.layers.attention.kv_shard_hooks import ( from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask from sglang.srt.layers.dcp.layout import get_dcp_lens from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run +from sglang.srt.mem_cache.layout.paged_view import paged_row_view from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( is_in_breakable_cuda_graph, @@ -1383,7 +1384,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): # Prepare KV cache inline k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) + kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1) # Get metadata metadata = ( @@ -1599,7 +1600,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): bs = forward_batch.batch_size k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) + kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1) q = q.to(self.data_type) diff --git a/python/sglang/srt/mem_cache/layout/paged_view.py b/python/sglang/srt/mem_cache/layout/paged_view.py new file mode 100644 index 0000000000..89d9c0d4aa --- /dev/null +++ b/python/sglang/srt/mem_cache/layout/paged_view.py @@ -0,0 +1,48 @@ +"""Paged views over per-slot KV buffers. + +Attention kernels take a paged ``[num_pages, page_size, ...]`` cache. These +helpers split a pool buffer's slot dim into pages from the buffer's own +strides, so a strided per-layer view keeps its slot stride at every page size. +""" + +import torch + + +def paged_view(flat: torch.Tensor, page_size: int) -> torch.Tensor: + """``[slots, ...]`` -> ``[slots // page_size, page_size, ...]``, as a view. + + Built with ``as_strided``, not ``view``: dim 1 must carry the slot stride + at every page size, and ``view`` gives a size-1 dim (``page_size == 1``) + the row stride instead, which differs from the slot stride whenever the + per-layer view is strided. + """ + num_slots = int(flat.shape[0]) + assert num_slots % page_size == 0, (num_slots, page_size) + slot_stride = flat.stride(0) + return flat.as_strided( + (num_slots // page_size, page_size, *flat.shape[1:]), + (slot_stride * page_size, slot_stride, *flat.stride()[1:]), + ) + + +def paged_kv_view( + buf: torch.Tensor, page_size: int, head_num: int, head_dim: int +) -> torch.Tensor: + """An MHA pool buffer as ``[num_pages, page_size, head_num, head_dim]``. + + A slot-major buffer has its rows shaped and its pages split from its own + strides; an HND buffer (``[num_pages, head_num, page_size, head_dim]``) is + already page-split and only has its head and token dims swapped. + """ + if buf.dim() <= 3: + return paged_view(buf.view(buf.shape[0], head_num, head_dim), page_size) + assert tuple(buf.shape[1:]) == (head_num, page_size, head_dim), ( + f"expected an HND buffer [pages, {head_num}, {page_size}, {head_dim}], " + f"got {tuple(buf.shape)}" + ) + return buf.permute(0, 2, 1, 3) + + +def paged_row_view(flat: torch.Tensor, page_size: int) -> torch.Tensor: + """``[slots, ...]`` -> ``[slots // page_size, page_size, row_elems]``, as a view.""" + return paged_view(flat.view(int(flat.shape[0]), -1), page_size) diff --git a/python/sglang/srt/models/utils.py b/python/sglang/srt/models/utils.py index e8a03a2f5a..80ada0c566 100644 --- a/python/sglang/srt/models/utils.py +++ b/python/sglang/srt/models/utils.py @@ -32,6 +32,7 @@ from sglang.kernels.ops.layernorm.norm import ( from sglang.srt.environ import envs from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.mem_cache.layout.paged_view import paged_kv_view from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_token_to_kv_pool @@ -347,11 +348,11 @@ def create_fused_set_kv_buffer_arg( key_cache = k_buffer value_cache = v_buffer else: - key_cache = k_buffer.view( - -1, page_size, layer.tp_k_head_num, layer.qk_head_dim + key_cache = paged_kv_view( + k_buffer, page_size, layer.tp_k_head_num, layer.qk_head_dim ) - value_cache = v_buffer.view( - -1, page_size, layer.tp_v_head_num, layer.v_head_dim + value_cache = paged_kv_view( + v_buffer, page_size, layer.tp_v_head_num, layer.v_head_dim ) return { "v": value.view(-1, layer.tp_v_head_num, layer.v_head_dim), diff --git a/test/registered/attention/test_trtllm_mha_stride_parity.py b/test/registered/attention/test_trtllm_mha_stride_parity.py new file mode 100644 index 0000000000..c9c65df62d --- /dev/null +++ b/test/registered/attention/test_trtllm_mha_stride_parity.py @@ -0,0 +1,173 @@ +"""trtllm_mha output must not depend on the KV cache's slot stride. + +`_reshape_paged_kv_cache` hands the trtllm-gen kernels a view whose in-page +token stride is the pool's SLOT stride. On a contiguous per-layer buffer that +stride is `heads * head_dim`; under the unified pool's token-major layout the +slot carries every layer, so the same view has a stride wider by +`2 * layer_num`. Shape and dtype are identical either way, so a kernel that +derives the page address from `page_size * heads * head_dim` instead of +reading `stride()` reads the wrong rows and returns wrong numbers -- with no +exception and no shape mismatch to catch it. + +This is the check the view-level tests cannot make +(`unit/layers/attention/test_trtllm_mha_paged_kv.py` proves the VIEW carries +the slot stride; it says nothing about what the kernel does with it). Identical +page contents go through a contiguous cache and through an over-strided view of +the same values, for both kernels the backend calls on SM100 (decode and +context) and for bf16 and fp8 KV; the outputs must match bit for bit. + +Runs on SM100 only: the trtllm-gen cubins are datacenter-Blackwell kernels, and +on SM90 / SM120 the backend dispatches to other kernels. + + python -m pytest test/registered/attention/test_trtllm_mha_stride_parity.py -v +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import is_sm100_supported +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200") + +_PAGES = 4 +_PAGE_SIZE = 16 +_HEADS = 4 +_HEAD_DIM = 128 +_BS = 2 +_Q_HEADS = 8 +# Stand-in for the unified pool's entry: every layer's K and V share the slot. +_LAYER_NUM = 8 + + +def _contiguous_cache(dtype, device): + """A plain per-layer buffer: slot stride == heads * head_dim.""" + slots = _PAGES * _PAGE_SIZE + return torch.randn(slots, _HEADS, _HEAD_DIM, device=device).to(dtype) + + +def _strided_like(src): + """The SAME values in a buffer whose slot stride spans a whole entry. + + `as_strided` over a wider backing reproduces exactly what a token-major + per-layer view looks like: rows `2 * _LAYER_NUM` apart, not packed. + """ + slots = src.shape[0] + entry = 2 * _LAYER_NUM * _HEADS * _HEAD_DIM + backing = torch.zeros(slots * entry, dtype=src.dtype, device=src.device) + view = backing.as_strided((slots, _HEADS, _HEAD_DIM), (entry, _HEAD_DIM, 1)) + view.copy_(src) + assert view.stride(0) == entry != src.stride(0) + assert torch.equal(view.view(torch.uint8), src.view(torch.uint8)) + return view + + +def _reshape(k, v): + from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend + + backend = TRTLLMHAAttnBackend.__new__(TRTLLMHAAttnBackend) + backend.page_size = _PAGE_SIZE + layer = SimpleNamespace(tp_k_head_num=_HEADS, tp_v_head_num=_HEADS) + return backend._reshape_paged_kv_cache(k, v, layer, _HEAD_DIM) + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +@unittest.skipUnless(is_sm100_supported(), "trtllm-gen kernels require SM100") +class TestTRTLLMHAStrideParity(unittest.TestCase): + def setUp(self): + torch.manual_seed(0) + self.workspace = torch.zeros( + 128 * 1024 * 1024, dtype=torch.uint8, device="cuda" + ) + self.block_tables = ( + torch.arange(_PAGES, dtype=torch.int32, device="cuda") + .repeat(_BS, 1) + .contiguous() + ) + + def _assert_stride_independent(self, run, kv_dtype): + k_contig = _contiguous_cache(kv_dtype, "cuda") + v_contig = _contiguous_cache(kv_dtype, "cuda") + want = run(_reshape(k_contig, v_contig)) + got = run(_reshape(_strided_like(k_contig), _strided_like(v_contig))) + # Bit-for-bit: a tolerance here would hide the bug. + self.assertTrue( + torch.equal(want, got), + f"trtllm_mha depends on the KV slot stride: " + f"max |diff| = {(want.float() - got.float()).abs().max().item()}", + ) + + def test_decode_output_is_identical_across_slot_strides(self): + import flashinfer + + for kv_dtype in (torch.bfloat16, torch.float8_e4m3fn): + with self.subTest(kv_dtype=kv_dtype): + q = torch.randn(_BS, _Q_HEADS, _HEAD_DIM, device="cuda").to( + torch.bfloat16 + ) + seq_lens = torch.tensor( + [_PAGES * _PAGE_SIZE - 5, _PAGES * _PAGE_SIZE], + dtype=torch.int32, + device="cuda", + ) + + def run(kv_cache): + return flashinfer.decode.trtllm_batch_decode_with_kv_cache( + query=q, + kv_cache=kv_cache, + workspace_buffer=self.workspace, + block_tables=self.block_tables, + seq_lens=seq_lens, + max_seq_len=_PAGES * _PAGE_SIZE, + bmm1_scale=_HEAD_DIM**-0.5, + bmm2_scale=1.0, + window_left=-1, + out_dtype=torch.bfloat16, + ) + + self._assert_stride_independent(run, kv_dtype) + + def test_context_output_is_identical_across_slot_strides(self): + import flashinfer + + q_lens = [5, 9] + kv_lens = [_PAGES * _PAGE_SIZE - 3, _PAGES * _PAGE_SIZE] + # The context kernel takes Q in the KV dtype. + for kv_dtype in (torch.bfloat16, torch.float8_e4m3fn): + with self.subTest(kv_dtype=kv_dtype): + q = torch.randn(sum(q_lens), _Q_HEADS, _HEAD_DIM, device="cuda").to( + kv_dtype + ) + seq_lens = torch.tensor(kv_lens, dtype=torch.int32, device="cuda") + cum_seq_lens_q = torch.tensor( + [0, q_lens[0], sum(q_lens)], dtype=torch.int32, device="cuda" + ) + cum_seq_lens_kv = torch.tensor( + [0, kv_lens[0], sum(kv_lens)], dtype=torch.int32, device="cuda" + ) + + def run(kv_cache): + return flashinfer.prefill.trtllm_batch_context_with_kv_cache( + query=q, + kv_cache=kv_cache, + workspace_buffer=self.workspace, + block_tables=self.block_tables, + seq_lens=seq_lens, + max_q_len=max(q_lens), + max_kv_len=max(kv_lens), + bmm1_scale=_HEAD_DIM**-0.5, + bmm2_scale=1.0, + batch_size=_BS, + cum_seq_lens_q=cum_seq_lens_q, + cum_seq_lens_kv=cum_seq_lens_kv, + window_left=-1, + out_dtype=torch.bfloat16, + ) + + self._assert_stride_independent(run, kv_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/attention/test_trtllm_mla_stride_parity.py b/test/registered/attention/test_trtllm_mla_stride_parity.py new file mode 100644 index 0000000000..3d83bb6375 --- /dev/null +++ b/test/registered/attention/test_trtllm_mla_stride_parity.py @@ -0,0 +1,156 @@ +"""The paged MLA decode kernels must not depend on the KV cache's slot stride. + +trtllm_mla, cutedsl_mla and tokenspeed_mla hand their decode kernels +`paged_row_view(k_cache, page_size).unsqueeze(1)`: `[pages, 1, page_size, +kv_cache_dim]` whose in-page token stride is the pool's SLOT stride. Under the +unified pool's token-major layout that stride spans every layer's latent row, +so a kernel that derives the page address from `page_size * kv_cache_dim` +instead of reading `stride()` reads the wrong rows with no error. Identical +latent rows go through a packed cache and through an over-strided view of the +same values; the outputs must match bit for bit. + +Runs on SM100 only, where these kernels exist. + + python -m pytest test/registered/attention/test_trtllm_mla_stride_parity.py -v +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.layout.paged_view import paged_row_view +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import is_sm100_supported, is_tokenspeed_mla_available +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=15, stage="base-b", runner_config="4-gpu-b200") + +_PAGES = 4 +_PAGE_SIZE = 64 +_KV_LORA_RANK = 512 +_QK_ROPE_HEAD_DIM = 64 +_QK_NOPE_HEAD_DIM = 128 +_KV_CACHE_DIM = _KV_LORA_RANK + _QK_ROPE_HEAD_DIM +_Q_HEADS = 16 +_BS = 2 +# Stand-in for the unified pool's entry: every layer's latent row shares the slot. +_LAYER_NUM = 61 + + +def _packed_cache(dtype): + slots = _PAGES * _PAGE_SIZE + return torch.randn(slots, 1, _KV_CACHE_DIM, device="cuda").to(dtype) + + +def _strided_like(src): + """The SAME rows in a buffer whose slot stride spans a whole entry.""" + slots = src.shape[0] + entry = _LAYER_NUM * _KV_CACHE_DIM + backing = torch.zeros(slots * entry, dtype=src.dtype, device=src.device) + view = backing.as_strided(src.shape, (entry, _KV_CACHE_DIM, 1)) + view.copy_(src) + assert view.stride(0) == entry != src.stride(0) + assert torch.equal(view.view(torch.uint8), src.view(torch.uint8)) + return view + + +def _paged(k_cache): + # What the MLA backends' forward_decode builds from the pool's key buffer. + return paged_row_view(k_cache, _PAGE_SIZE).unsqueeze(1) + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +@unittest.skipUnless(is_sm100_supported(), "the paged MLA kernels require SM100") +class TestPagedMLADecodeStrideParity(unittest.TestCase): + def setUp(self): + torch.manual_seed(0) + self.block_tables = ( + torch.arange(_PAGES, dtype=torch.int32, device="cuda") + .repeat(_BS, 1) + .contiguous() + ) + self.seq_lens = torch.tensor( + [_PAGES * _PAGE_SIZE - 7, _PAGES * _PAGE_SIZE], + dtype=torch.int32, + device="cuda", + ) + + def _assert_stride_independent(self, run, kv_dtype): + packed = _packed_cache(kv_dtype) + want = run(_paged(packed)) + got = run(_paged(_strided_like(packed))) + # Bit-for-bit: a tolerance here would hide the bug. + self.assertTrue( + torch.equal(want, got), + f"MLA decode depends on the KV slot stride: " + f"max |diff| = {(want.float() - got.float()).abs().max().item()}", + ) + + def test_flashinfer_mla_decode_is_identical_across_slot_strides(self): + import flashinfer + + workspace = torch.zeros(128 * 1024 * 1024, dtype=torch.uint8, device="cuda") + # trtllm-gen is trtllm_mla's kernel; cute-dsl is cutedsl_mla's. + for backend, kv_dtype in ( + ("trtllm-gen", torch.bfloat16), + ("trtllm-gen", torch.float8_e4m3fn), + ("cute-dsl", torch.bfloat16), + ): + with self.subTest(backend=backend, kv_dtype=kv_dtype): + q = torch.randn(_BS, 1, _Q_HEADS, _KV_CACHE_DIM, device="cuda").to( + kv_dtype + ) + extra = {} if backend == "trtllm-gen" else {"backend": backend} + + def run(kv_cache): + return flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=q, + kv_cache=kv_cache, + workspace_buffer=workspace, + qk_nope_head_dim=_QK_NOPE_HEAD_DIM, + kv_lora_rank=_KV_LORA_RANK, + qk_rope_head_dim=_QK_ROPE_HEAD_DIM, + block_tables=self.block_tables, + seq_lens=self.seq_lens, + max_seq_len=_PAGES * _PAGE_SIZE, + bmm1_scale=_KV_CACHE_DIM**-0.5, + **extra, + ) + + self._assert_stride_independent(run, kv_dtype) + + def test_tokenspeed_mla_decode_is_identical_across_slot_strides(self): + if not is_tokenspeed_mla_available(): + self.skipTest("tokenspeed_mla is not installed") + import tokenspeed_mla + + from sglang.srt.layers.attention.tokenspeed_mla_backend import ( + _get_tokenspeed_workspace, + ) + + # The workspace is sized for the DCP-gathered head count. + with get_parallel().override(attn_dcp_size=1): + workspace = _get_tokenspeed_workspace( + torch.device("cuda"), _Q_HEADS, _KV_LORA_RANK + ) + kv_dtype = torch.float8_e4m3fn + q = torch.randn(_BS, 1, _Q_HEADS, _KV_CACHE_DIM, device="cuda").to(kv_dtype) + + def run(kv_cache): + return tokenspeed_mla.tokenspeed_mla_decode( + query=q, + kv_cache=kv_cache, + workspace_buffer=workspace, + kv_lora_rank=_KV_LORA_RANK, + qk_rope_head_dim=_QK_ROPE_HEAD_DIM, + block_tables=self.block_tables, + seq_lens=self.seq_lens, + max_seq_len=_PAGES * _PAGE_SIZE, + softmax_scale=_KV_CACHE_DIM**-0.5, + ) + + self._assert_stride_independent(run, kv_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/attention/test_trtllm_mha_paged_kv.py b/test/registered/unit/layers/attention/test_trtllm_mha_paged_kv.py new file mode 100644 index 0000000000..a55795b3eb --- /dev/null +++ b/test/registered/unit/layers/attention/test_trtllm_mha_paged_kv.py @@ -0,0 +1,94 @@ +"""trtllm_mha's paged K/V views keep the pool's slot stride. + +``_reshape_paged_kv_cache`` turns a per-layer ``[slots, ...]`` buffer into the +``[pages, heads, page_size, head_dim]`` layout the trtllm kernels take. Its +nvfp4 callers also hand it block-scale rows that arrive FLAT +(``[slots, heads * head_dim // 16]``), so the row has to be shaped before the +pages are split: shaping it afterwards, or splitting with ``.view()``, +re-derives the size-1 page dimension's stride from the row at +``page_size == 1``. On a strided buffer (the unified pool's slots are a whole +entry apart, wider than one row) that stride is then wrong. + +An HND pool hands over page-major ``[pages, heads, page_size, head_dim]`` +buffers instead, which are already the kernel layout: they must reach the +kernel as themselves, head for head and token for token. + +CPU-only. + + python -m pytest test/registered/unit/layers/attention/test_trtllm_mha_paged_kv.py -v +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _strided_rows(num_slots, row_shape, slot_stride): + # Distinct values, so equal values mean equal addresses. + backing = torch.arange(num_slots * slot_stride, dtype=torch.float32) + return backing.as_strided( + (num_slots, *row_shape), (slot_stride, *torch.empty(row_shape).stride()) + ) + + +class TestTRTLLMHAPagedKV(unittest.TestCase): + def test_reshape_keeps_slot_stride_for_packed_and_flat_rows(self): + H, D, pages = 4, 32, 3 + slot_stride = 3 * H * D + layer = SimpleNamespace(tp_k_head_num=H, tp_v_head_num=H) + # 16 is the smallest page size trtllm_mha runs at; at 1, `.view()` + # would give the size-1 page dim the row stride. + for page_size in (1, 16): + backend = TRTLLMHAAttnBackend.__new__(TRTLLMHAAttnBackend) + backend.page_size = page_size + # (row shape handed in, head_dim argument): nvfp4 packed K/V rows, + # then the flat block-scale rows. + for row_shape, head_dim in ( + ((H, D // 2), D // 2), + ((H * D // 16,), D // 16), + ): + with self.subTest(page_size=page_size, row_shape=row_shape): + k = _strided_rows(pages * page_size, row_shape, slot_stride) + v = _strided_rows(pages * page_size, row_shape, slot_stride) + outs = backend._reshape_paged_kv_cache(k, v, layer, head_dim) + for flat, out in zip((k, v), outs): + self.assertEqual( + tuple(out.shape), (pages, H, page_size, head_dim) + ) + self.assertEqual(out.stride(0), page_size * slot_stride) + self.assertEqual(out.stride(2), slot_stride) + for token in range(pages * page_size): + page, slot = divmod(token, page_size) + self.assertTrue( + torch.equal( + out[page, :, slot], + flat[token].reshape(H, head_dim), + ) + ) + + def test_reshape_hands_the_hnd_pool_buffer_through(self): + H, D, pages, page_size = 4, 32, 3, 16 + backend = TRTLLMHAAttnBackend.__new__(TRTLLMHAAttnBackend) + backend.page_size = page_size + layer = SimpleNamespace(tp_k_head_num=H, tp_v_head_num=H) + # Distinct values, so equal values mean equal (page, head, token). + k = torch.arange(pages * H * page_size * D, dtype=torch.float32).view( + pages, H, page_size, D + ) + v = k + 0.5 + outs = backend._reshape_paged_kv_cache(k, v, layer, D) + for buf, out in zip((k, v), outs): + self.assertEqual(out.shape, buf.shape) + self.assertEqual(out.stride(), buf.stride()) + self.assertEqual(out.data_ptr(), buf.data_ptr()) + self.assertTrue(torch.equal(out, buf)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_flashattention_paged_mha.py b/test/registered/unit/layers/test_flashattention_paged_mha.py index 7a1799be56..8ed2849418 100644 --- a/test/registered/unit/layers/test_flashattention_paged_mha.py +++ b/test/registered/unit/layers/test_flashattention_paged_mha.py @@ -21,6 +21,10 @@ with patch.dict( from sglang.srt.layers.attention.flashattention_backend import ( FlashAttentionBackend, ) + from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( + UnquantizedKVCacheMethod, + ) + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -36,6 +40,28 @@ def _backend(): return backend +def _mha_pool(k_buffer, v_buffer, *, page_size): + pool = MHATokenToKVPool.__new__(MHATokenToKVPool) + pool.page_size = page_size + pool.start_layer = 0 + pool.layer_transfer_counter = None + pool.quant_method = UnquantizedKVCacheMethod() + pool.dtype = pool.store_dtype = k_buffer.dtype + pool.k_buffer = [k_buffer] + pool.v_buffer = [v_buffer] + return pool + + +def _layer(): + return SimpleNamespace( + layer_id=0, + tp_k_head_num=2, + tp_v_head_num=2, + head_dim=16, + v_head_dim=16, + ) + + class TestFlashAttentionPagedMHA(unittest.TestCase): def test_get_paged_mha_kv_cache_supports_head_groups(self): backend = _backend() @@ -63,6 +89,61 @@ class TestFlashAttentionPagedMHA(unittest.TestCase): self.assertEqual(key_cache.shape, (16, 1, 1, 16)) self.assertEqual(value_cache.shape, (16, 1, 1, 16)) + def test_get_paged_mha_kv_cache_refuses_head_groups_on_a_strided_pool(self): + # A unified pool's slots are a whole ENTRY apart, and no view of the + # head dim can regroup pages over that stride: refuse it loudly. + layer = _layer() + H, D, slots = layer.tp_k_head_num, layer.head_dim, 8 + entry = 64 * H * D # every layer's K and V share the slot + backing = torch.zeros(slots * entry) + strided = backing.as_strided((slots, H, D), (entry, D, 1)) + self.assertFalse(strided.is_contiguous()) + + backend = _backend() + backend.token_to_kv_pool = SimpleNamespace( + get_kv_buffer=Mock(return_value=(strided, strided)) + ) + + with self.assertRaises(AssertionError) as caught: + backend.get_paged_mha_kv_cache(layer, head_group_num=2) + self.assertIn("contiguous paged KV", str(caught.exception)) + + key_cache, _ = backend.get_paged_mha_kv_cache(layer) + self.assertEqual(key_cache.shape, (slots, 1, H, D)) + + def test_get_paged_mha_kv_cache_pages_by_the_backend_page_size(self): + # Pages follow the backend's page size even when the pool pages at a + # multiple of it, as a draft pool does under decode context parallelism. + backend = _backend() + backend.page_size = 2 + k = torch.arange(8 * 2 * 16, dtype=torch.float16).view(8, 2, 16) + backend.token_to_kv_pool = _mha_pool(k, k + 1, page_size=4) + + key_cache, value_cache = backend.get_paged_mha_kv_cache(_layer()) + + self.assertEqual(key_cache.shape, (4, 2, 2, 16)) + self.assertEqual(value_cache.shape, (4, 2, 2, 16)) + self.assertTrue(torch.equal(key_cache[1, 0], k[2])) + + def test_get_paged_mha_kv_cache_reads_the_hnd_pool_by_head_and_token(self): + # An HND pool hands over [num_pages, heads, page_size, head_dim] + # buffers; the paged cache must index them as [page, token, head]. + backend = _backend() + backend.page_size = 4 + k = torch.arange(3 * 2 * 4 * 16, dtype=torch.float16).view(3, 2, 4, 16) + backend.token_to_kv_pool = _mha_pool(k, k + 1, page_size=4) + + key_cache, _ = backend.get_paged_mha_kv_cache(_layer()) + + self.assertEqual(key_cache.shape, (3, 4, 2, 16)) + self.assertEqual(key_cache.data_ptr(), k.data_ptr()) + for page in range(3): + for head in range(2): + for token in range(4): + self.assertTrue( + torch.equal(key_cache[page, token, head], k[page, head, token]) + ) + def test_prepare_paged_mha_query_reuses_fa_scaling_policy(self): backend = _backend() layer = SimpleNamespace( diff --git a/test/registered/unit/mem_cache/test_paged_view.py b/test/registered/unit/mem_cache/test_paged_view.py new file mode 100644 index 0000000000..1bf9cadb6f --- /dev/null +++ b/test/registered/unit/mem_cache/test_paged_view.py @@ -0,0 +1,96 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""``paged_view`` regroups ``[slots, ...]`` into ``[pages, page_size, ...]``. + +A pool's per-layer buffer may be a strided view: consecutive slots are a whole +entry apart, not one row. The page split must then keep that slot stride on +dim 1 at every page size; ``.view()`` does not, since at ``page_size == 1`` it +gives the size-1 page dim the ROW stride. These tests pin the strides the +helper produces, at both page sizes, against the buffer addresses they must +reproduce. + +CPU-only. + + python -m pytest test/registered/unit/mem_cache/test_paged_view.py -v +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.layout.paged_view import ( + paged_kv_view, + paged_row_view, + paged_view, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestPagedView(unittest.TestCase): + def test_contiguous_matches_plain_view(self): + N, ps, H, D = 12, 4, 2, 16 + flat = torch.zeros(N, H, D) + out = paged_view(flat, ps) + self.assertEqual(tuple(out.shape), (N // ps, ps, H, D)) + self.assertEqual(out.stride(), flat.view(-1, ps, H, D).stride()) + self.assertEqual(out.data_ptr(), flat.data_ptr()) + + def test_strided_slots_keep_their_stride(self): + # At ps == 1 the size-1 page dim must still carry the slot stride + # (a whole entry), where `.view()` would give it the row stride. + for ps in (1, 4): + with self.subTest(page_size=ps): + N, H, D, E = 12, 2, 16, 2 * 16 + 96 + flat = torch.zeros(N * E).as_strided((N, H, D), (E, D, 1)) + out = paged_view(flat, ps) + self.assertEqual(tuple(out.shape), (N // ps, ps, H, D)) + self.assertEqual(tuple(out.stride()), (ps * E, E, D, 1)) + for t in range(N): + self.assertEqual( + out[t // ps, t % ps].data_ptr(), flat[t].data_ptr() + ) + rows = paged_row_view(flat, ps) + self.assertEqual(tuple(rows.shape), (N // ps, ps, H * D)) + self.assertEqual(tuple(rows.stride()), (ps * E, E, 1)) + + def test_rejects_partial_pages(self): + with self.assertRaises(AssertionError): + paged_view(torch.zeros(10, 2, 16), 4) + + +class TestPagedKVView(unittest.TestCase): + def test_hnd_buffer_is_indexed_by_page_token_head(self): + pages, H, ps, D = 3, 4, 16, 8 + hnd = torch.arange(pages * H * ps * D, dtype=torch.float32).view( + pages, H, ps, D + ) + out = paged_kv_view(hnd, ps, H, D) + self.assertEqual(tuple(out.shape), (pages, ps, H, D)) + self.assertEqual(out.data_ptr(), hnd.data_ptr()) + # HND element (page, head, token, dim) sits at this flat offset. + for p in range(pages): + for t in range(ps): + for h in range(H): + want = ((p * H + h) * ps + t) * D + self.assertEqual(int(out[p, t, h, 0]), want) + + def test_hnd_buffer_of_another_geometry_is_refused(self): + with self.assertRaises(AssertionError): + paged_kv_view(torch.zeros(3, 16, 4, 8), 16, 4, 8) + + +if __name__ == "__main__": + unittest.main()