diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index ee34067682..2635f2b698 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -14,7 +14,18 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import ( rotate_activation, ) from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod -from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp +from sglang.srt.layers.attention.dsa.paged_mqa_logits_backend import ( + DSAPagedMQALogitsBackend, +) +from sglang.srt.layers.attention.dsa.utils import ( + aiter_can_use_preshuffle_paged_mqa, + dsa_use_prefill_cp, +) +from sglang.srt.layers.attention.mqa_logits_utils import ( + MQA_LOGITS_BYTES_PER_ELEM, + MQA_LOGITS_MAX_BYTES_ROCM, + mqa_logits_rows_per_chunk, +) from sglang.srt.layers.layernorm import LayerNorm from sglang.srt.layers.utils import MultiPlatformOp from sglang.srt.utils import add_prefix, ceil_align, is_cuda, is_hip, is_npu @@ -46,7 +57,7 @@ from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( is_in_breakable_cuda_graph, ) -from sglang.srt.runtime_context import get_device +from sglang.srt.runtime_context import get_device, get_exec if TYPE_CHECKING: from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool @@ -110,6 +121,11 @@ class IndexerKPool(MultiPlatformOp): f"index_kpool ({self.index_kpool}) must divide page_size (64)" ) + # resolve() pins ROCm to AITER and lets --dsa-paged-mqa-logits-backend reach this path + self.paged_mqa_logits_backend = DSAPagedMQALogitsBackend.resolve( + get_exec().kernel.dsa_paged_mqa_logits_backend + ) + self.index_kpool_compress_ape = nn.Parameter( torch.zeros(self.index_kpool, self.head_dim, dtype=torch.float32) ) @@ -808,12 +824,66 @@ class IndexerKPool(MultiPlatformOp): return None, attn_metadata.topk_indices_offset, None return None, None, None + @staticmethod + def _fp8_mqa_logits( + q_fp8: torch.Tensor, + k_fp8: torch.Tensor, + k_scale: torch.Tensor, + weights: torch.Tensor, + starts: torch.Tensor, + ends: torch.Tensor, + *, + clean_logits: bool, + ) -> torch.Tensor: + """Ragged MQA logits: DeepGEMM on CUDA, AITER's Triton kernel on ROCm.""" + if is_hip(): + from aiter.ops.triton.fp8_mqa_logits import fp8_mqa_logits + + # above 2 GiB of fp32 logits the kernel miscompiles and abort()s the + # process (ROCm/aiter#5114); rows are independent, so block the queries + num_q, num_k = q_fp8.shape[0], k_fp8.shape[0] + rows_per_call = mqa_logits_rows_per_chunk( + num_rows=num_q, + row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM, + budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM, + ) + if rows_per_call is None: + return fp8_mqa_logits( + q_fp8, + k_fp8, + k_scale, + weights, + starts, + ends, + clean_logits=clean_logits, + ) + logits = torch.empty( + (num_q, num_k), dtype=torch.float32, device=q_fp8.device + ) + for i in range(0, num_q, rows_per_call): + rows = slice(i, i + rows_per_call) + logits[rows] = fp8_mqa_logits( + q_fp8[rows], + k_fp8, + k_scale, + weights[rows], + starts[rows], + ends[rows], + clean_logits=clean_logits, + ) + return logits + + return deep_gemm.fp8_mqa_logits( + q_fp8, (k_fp8, k_scale), weights, starts, ends, clean_logits=clean_logits + ) + @staticmethod def _should_use_tilelang_paged_mqa_logits(q_fp8: torch.Tensor) -> bool: + # q_fp8 is [tokens, heads, dim], without the next_n dim if not is_cuda(): return False arch_major, _ = torch.cuda.get_device_capability(q_fp8.device) - num_heads = q_fp8.shape[2] + num_heads = q_fp8.shape[1] return arch_major == 9 and num_heads not in (32, 64) def _get_topk_paged( @@ -850,7 +920,7 @@ class IndexerKPool(MultiPlatformOp): if n_real < num_q_padded: q_fp8 = q_fp8[:n_real] weights = weights[:n_real] - q_fp8 = q_fp8.unsqueeze(1) # the next_n dim is 1 now + # aiter_paged_mqa_logits adds the next_n dim itself; the other two want it already there assert len(kv_cache_fp8.shape) == 2 block_kv = 64 num_heads_kv = 1 @@ -860,7 +930,11 @@ class IndexerKPool(MultiPlatformOp): ) assert len(weights.shape) == 3 weights = weights.squeeze(2) - use_tilelang_paged_mqa = self._should_use_tilelang_paged_mqa_logits(q_fp8) + use_aiter_paged_mqa = self.paged_mqa_logits_backend.is_aiter() + use_tilelang_paged_mqa = ( + not use_aiter_paged_mqa + and self._should_use_tilelang_paged_mqa_logits(q_fp8) + ) pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = ( self._get_kpool_decode_metadata( @@ -868,17 +942,40 @@ class IndexerKPool(MultiPlatformOp): block_tables, seqlens_32, blocksize, - build_schedule_metadata=not use_tilelang_paged_mqa, + build_schedule_metadata=not ( + use_aiter_paged_mqa or use_tilelang_paged_mqa + ), ) ) pool_max_seq_len = pool_block_tables.shape[1] * blocksize - if use_tilelang_paged_mqa: + if use_aiter_paged_mqa: + # kpool_fp8_index writes in AITER's tile order, so a row-major reader would misread the cache + if not aiter_can_use_preshuffle_paged_mqa(): + raise RuntimeError( + "The ROCm k-pool indexer needs AITER's preshuffle paged-MQA " + "kernel. It requires Triton >= 3.5.0, or " + "AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1 with the AOT gluon " + "artifacts present in the image." + ) + from sglang.kernels.ops.attention.dsa import aiter_paged_mqa_logits + + logits = aiter_paged_mqa_logits( + q_fp8, + kv_cache_fp8, + weights, + pool_seqlens, + pool_block_tables, + pool_max_seq_len, + preshuffle=True, + kv_block_size=block_kv, + ) + elif use_tilelang_paged_mqa: from sglang.kernels.ops.attention.dsa.tilelang_kernel import ( tilelang_fp8_paged_mqa_logits, ) logits = tilelang_fp8_paged_mqa_logits( - q_fp8, + q_fp8.unsqueeze(1), kv_cache_fp8, weights, pool_seqlens, @@ -889,7 +986,7 @@ class IndexerKPool(MultiPlatformOp): ) else: logits = deep_gemm.fp8_paged_mqa_logits( - q_fp8, + q_fp8.unsqueeze(1), kv_cache_fp8, weights, pool_context_lens, @@ -967,9 +1064,10 @@ class IndexerKPool(MultiPlatformOp): scale_out=k_scale, ) k_fp8 = k_u8.view(torch.float8_e4m3fn) - logits = deep_gemm.fp8_mqa_logits( + logits = self._fp8_mqa_logits( q_fp8[:n_real].contiguous(), - (k_fp8.contiguous(), k_scale.contiguous()), + k_fp8.contiguous(), + k_scale.contiguous(), weights[:n_real].contiguous(), ks_per_q, ke_per_q, @@ -1199,9 +1297,10 @@ class IndexerKPool(MultiPlatformOp): and zero_starts_by_batch[i] is not None else torch.zeros((q_len,), dtype=torch.int32, device=q_fp8.device) ) - local_logits = deep_gemm.fp8_mqa_logits( + local_logits = self._fp8_mqa_logits( q_fp8[q_slice].contiguous(), - (k_fp8.contiguous(), k_scale.contiguous()), + k_fp8.contiguous(), + k_scale.contiguous(), weights[q_slice].contiguous(), row_starts, local_pool_lens, @@ -1575,7 +1674,7 @@ class IndexerKPool(MultiPlatformOp): if not return_indices: return None - if is_cuda(): + if is_cuda() or is_hip(): if ( forward_batch.forward_mode.is_decode_or_idle() or forward_batch.forward_mode.is_target_verify() @@ -1603,5 +1702,7 @@ class IndexerKPool(MultiPlatformOp): kpool_extend_cache=kpool_extend_cache, ) else: - raise NotImplementedError("kpool indexer is only supported on CUDA") + raise NotImplementedError( + "kpool indexer is only supported on CUDA and ROCm" + ) return topk_result diff --git a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py index db8d3a88a3..7c8cc44e6f 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py @@ -4,11 +4,50 @@ import torch import triton import triton.language as tl +from sglang.srt.layers.attention.dsa.utils import ( + INDEXER_K_CACHE_PRESHUFFLE_TILE, + aiter_can_use_preshuffle_paged_mqa, +) + BLOCK_SIZE_K = 64 INDEX_HEAD_DIM = 128 KPOOL_SCORE_DTYPES = (torch.float16, torch.bfloat16, torch.float32) +def _preshuffle_tile() -> int: + """Tile width for the index-K cache, or 0 to keep the row-major layout.""" + return ( + INDEXER_K_CACHE_PRESHUFFLE_TILE if aiter_can_use_preshuffle_paged_mqa() else 0 + ) + + +@triton.jit +def _kpool_cache_k_offsets( + page, + slot, + cols, + BUF_NUMEL_PER_PAGE: tl.constexpr, + HEAD_DIM: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, +): + """Offsets of one pooled token's index-K row; must match `_set_k_and_s_triton_kernel`, + which lays out the non-pooled cache for the same aiter kernel.""" + if PRESHUFFLE_TILE: + tile = PRESHUFFLE_TILE + token_tile_id = slot // tile + token_in_tile = slot % tile + col_tile_id = cols // tile + col_in_tile = cols % tile + return ( + page * BUF_NUMEL_PER_PAGE + + token_tile_id * (tile * HEAD_DIM) + + col_tile_id * (tile * tile) + + token_in_tile * tile + + col_in_tile + ) + return page * BUF_NUMEL_PER_PAGE + slot * HEAD_DIM + cols + + def kpool_max_closed_pools(num_draft_tokens: int, pool_size: int) -> int: return (num_draft_tokens + pool_size - 1) // pool_size @@ -59,6 +98,7 @@ def gather_index_k_scale_prefix_into( scale_out, PAGE_SIZE=pool.page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], + PRESHUFFLE_TILE=_preshuffle_tile(), HEAD_DIM=INDEX_HEAD_DIM, S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), @@ -76,6 +116,7 @@ def _gather_index_k_scale_prefix_into_kernel( BUF_NUMEL_PER_PAGE: tl.constexpr, HEAD_DIM: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, BLOCK_D: tl.constexpr, ): token_id = tl.program_id(0) @@ -85,7 +126,14 @@ def _gather_index_k_scale_prefix_into_kernel( offs = tl.arange(0, BLOCK_D) mask = offs < HEAD_DIM - src_k_offsets = page * BUF_NUMEL_PER_PAGE + token_offset_in_page * HEAD_DIM + offs + src_k_offsets = _kpool_cache_k_offsets( + page, + token_offset_in_page, + offs, + BUF_NUMEL_PER_PAGE, + HEAD_DIM, + PRESHUFFLE_TILE, + ) dst_k_offsets = token_id * HEAD_DIM + offs k = tl.load(buf_u8_ptr + src_k_offsets, mask=mask) tl.store(k_out_ptr + dst_k_offsets, k, mask=mask) @@ -725,6 +773,7 @@ def kpool_softmax_rotate_write_cache( ape.stride(0), PAGE_SIZE=pool.page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], + PRESHUFFLE_TILE=_preshuffle_tile(), POOL_SIZE=slot_k.shape[1], HEAD_DIM=slot_k.shape[2], S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim, @@ -818,6 +867,7 @@ def kpool_decode_update_and_maybe_write_cache( REQ_POOL_SIZE=tail_k.shape[0], PAGE_SIZE=pool.page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], + PRESHUFFLE_TILE=_preshuffle_tile(), POOL_SIZE=pool.index_kpool, TAIL_SIZE=tail_k.shape[1], HEAD_DIM=tail_k.shape[2], @@ -872,6 +922,7 @@ def _kpool_softmax_rotate_write_cache_kernel( POOL_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, ROUND_SCALE: tl.constexpr, HAS_WRITE_MASK: tl.constexpr, RETURN_COMPRESSED: tl.constexpr, @@ -947,10 +998,13 @@ def _kpool_softmax_rotate_write_cache_kernel( loc = tl.load(loc_ptr + row, mask=do_write, other=0) loc_page_index = loc // PAGE_SIZE loc_token_offset_in_page = loc % PAGE_SIZE - out_k_offsets = ( - loc_page_index * BUF_NUMEL_PER_PAGE - + loc_token_offset_in_page * HEAD_DIM - + offs + out_k_offsets = _kpool_cache_k_offsets( + loc_page_index, + loc_token_offset_in_page, + offs, + BUF_NUMEL_PER_PAGE, + HEAD_DIM, + PRESHUFFLE_TILE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 @@ -1000,6 +1054,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( HEAD_DIM: tl.constexpr, BLOCK_TABLE_COLS: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, ROUND_SCALE: tl.constexpr, BLOCK_D: tl.constexpr, SLOTS_PER_PAGE: tl.constexpr, @@ -1112,10 +1167,13 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( ) loc_page_index = packed_page.to(tl.int64) loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE - out_k_offsets = ( - loc_page_index * BUF_NUMEL_PER_PAGE - + loc_token_offset_in_page * HEAD_DIM - + offs + out_k_offsets = _kpool_cache_k_offsets( + loc_page_index, + loc_token_offset_in_page, + offs, + BUF_NUMEL_PER_PAGE, + HEAD_DIM, + PRESHUFFLE_TILE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 @@ -1175,6 +1233,7 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel( TAIL_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, ROUND_SCALE: tl.constexpr, HAS_WRITE_MASK: tl.constexpr, BLOCK_D: tl.constexpr, @@ -1222,8 +1281,13 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel( loc = tl.load(loc_ptr + row) loc_page_index = loc // SLOTS_PER_PAGE loc_token_offset_in_page = loc % SLOTS_PER_PAGE - out_k_offsets = ( - loc_page_index * BUF_NUMEL_PER_PAGE + loc_token_offset_in_page * HEAD_DIM + offs + out_k_offsets = _kpool_cache_k_offsets( + loc_page_index, + loc_token_offset_in_page, + offs, + BUF_NUMEL_PER_PAGE, + HEAD_DIM, + PRESHUFFLE_TILE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 @@ -1290,6 +1354,7 @@ def kpool_assemble_softmax_rotate_write_cache( tail_k.stride(1), ape.stride(0), BUF_NUMEL_PER_PAGE=buf.shape[1], + PRESHUFFLE_TILE=_preshuffle_tile(), POOL_SIZE=pool_size, TAIL_SIZE=tail_k.shape[1], HEAD_DIM=INDEX_HEAD_DIM, @@ -1537,6 +1602,7 @@ def _kpool_write_tail_and_maybe_compress_kernel( SLOTS_PER_PAGE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, + PRESHUFFLE_TILE: tl.constexpr, ROUND_SCALE: tl.constexpr, HAS_EFFECTIVE_N: tl.constexpr, MAX_CLOSED_POOLS: tl.constexpr, @@ -1601,10 +1667,13 @@ def _kpool_write_tail_and_maybe_compress_kernel( loc = tl.load(write_loc_ptr + b * write_loc_stride_0 + p) loc_page_index = loc // SLOTS_PER_PAGE loc_token_offset_in_page = loc % SLOTS_PER_PAGE - out_k_offsets = ( - loc_page_index * BUF_NUMEL_PER_PAGE - + loc_token_offset_in_page * HEAD_DIM - + offs + out_k_offsets = _kpool_cache_k_offsets( + loc_page_index, + loc_token_offset_in_page, + offs, + BUF_NUMEL_PER_PAGE, + HEAD_DIM, + PRESHUFFLE_TILE, ) out_s_offset = ( loc_page_index * BUF_NUMEL_PER_PAGE // 4 @@ -1694,6 +1763,7 @@ def kpool_write_tail_and_maybe_compress( BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), SLOTS_PER_PAGE=slots_per_page, BUF_NUMEL_PER_PAGE=buf.shape[1], + PRESHUFFLE_TILE=_preshuffle_tile(), S_OFFSET_NBYTES_IN_PAGE=slots_per_page * pool.index_head_dim, ROUND_SCALE=round_scale, HAS_EFFECTIVE_N=effective_n_per_batch is not None, diff --git a/test/registered/unit/layers/attention/test_dsa_mqa_logits_chunking.py b/test/registered/unit/layers/attention/test_dsa_mqa_logits_chunking.py index 4d9e0a703d..5d37c5e123 100644 --- a/test/registered/unit/layers/attention/test_dsa_mqa_logits_chunking.py +++ b/test/registered/unit/layers/attention/test_dsa_mqa_logits_chunking.py @@ -5,15 +5,21 @@ On ROCm the `[num_q x num_k]` fp32 logits tensor goes to aiter's `fp8_mqa_logits`, which only compiles below 2 GiB, so the budget that decides chunking is a correctness bound there and not only an out-of-memory guard. -The measured memory budget is stubbed: it is the only input the limit has to -beat, and stubbing it keeps these tests on CPU. +`Indexer` decides that with a budget; `IndexerKPool` splits the query rows +around its aiter call. Both are stubbed down to CPU-sized inputs. """ +import sys +import types +from unittest import mock + import pytest torch = pytest.importorskip("torch") +from sglang.srt.layers.attention.dsa import dsa_indexer_kpool # noqa: E402 from sglang.srt.layers.attention.mqa_logits_utils import ( # noqa: E402 + MQA_LOGITS_BYTES_PER_ELEM, MQA_LOGITS_MAX_BYTES_ROCM, mqa_logits_should_chunk, ) @@ -52,6 +58,60 @@ def test_off_rocm_the_budget_is_untouched(): assert _decide(16_384, 32_768, is_hip=False) == (False, HUGE_MEM_BUDGET) +def _kpool_mqa_logits(num_q, num_k, cap_bytes): + """Run `IndexerKPool._fp8_mqa_logits` over a stub that returns each row's + index plus its `starts`, so a misordered reassembly or an unsliced per-row + argument shows up in the logits.""" + rows_per_call = [] + + def _kernel(q_fp8, k_fp8, k_scale, weights, starts, ends, *, clean_logits): + rows = q_fp8.shape[0] + assert weights.shape[0] == rows + assert starts.shape[0] == rows + assert ends.shape[0] == rows + rows_per_call.append(rows) + row_value = q_fp8[:, 0, 0] + starts + return row_value.unsqueeze(1).expand(-1, k_fp8.shape[0]).contiguous() + + aiter_kernel_module = types.ModuleType("aiter.ops.triton.fp8_mqa_logits") + aiter_kernel_module.fp8_mqa_logits = _kernel + stub_modules = { + name: types.ModuleType(name) + for name in ("aiter", "aiter.ops", "aiter.ops.triton") + } + stub_modules["aiter.ops.triton.fp8_mqa_logits"] = aiter_kernel_module + + q_fp8 = torch.arange(num_q, dtype=torch.float32).view(num_q, 1, 1).expand(-1, 4, 8) + k_fp8 = torch.zeros(num_k, 8, dtype=torch.float32) + with ( + mock.patch.dict(sys.modules, stub_modules), + mock.patch.object(dsa_indexer_kpool, "is_hip", lambda: True), + mock.patch.object(dsa_indexer_kpool, "MQA_LOGITS_MAX_BYTES_ROCM", cap_bytes), + ): + logits = dsa_indexer_kpool.IndexerKPool._fp8_mqa_logits( + q_fp8, + k_fp8, + torch.ones(num_k), + torch.ones(num_q, 4), + torch.arange(num_q, dtype=torch.float32) * 10, + torch.full((num_q,), num_k, dtype=torch.int32), + clean_logits=True, + ) + return logits, rows_per_call + + +def test_kpool_splits_the_query_rows_to_stay_under_the_aiter_cap(): + """Above 2 GiB of logits the aiter kernel abort()s the process instead of + raising, so the k-pool wrapper must never hand it a larger tensor.""" + num_q, num_k = 8, 4 + row_bytes = num_k * MQA_LOGITS_BYTES_PER_ELEM + logits, rows_per_call = _kpool_mqa_logits(num_q, num_k, 3 * row_bytes + 1) + + assert rows_per_call == [3, 3, 2] + rows = torch.arange(num_q, dtype=torch.float32) + assert torch.equal(logits, (rows + rows * 10).unsqueeze(1).expand(-1, num_k)) + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py new file mode 100644 index 0000000000..a48d873dff --- /dev/null +++ b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py @@ -0,0 +1,59 @@ +"""The k-pool reader must see the index-K cache the way the non-pooled writer lays it out.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.kernels.ops.attention.dsa.index_buf_accessor import SetKAndS, _is_fp8_fnuz +from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( + INDEX_HEAD_DIM, + gather_index_k_scale_prefix_into, +) +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=15, suite="stage-b-test-1-gpu-small-amd-mi35x") + +PAGE_SIZE = 64 + + +@unittest.skipUnless(torch.cuda.is_available(), "Test requires a GPU") +class TestKpoolPreshuffledCacheLayout(CustomTestCase): + def test_gather_reads_back_what_set_k_and_s_wrote(self): + """A layout mismatch between the two writers never raises; it silently + returns the wrong top-k.""" + torch.manual_seed(0) + pool = SimpleNamespace(page_size=PAGE_SIZE) + num_pages, seq_len = 2, PAGE_SIZE + 5 + buf = torch.zeros( + num_pages, + PAGE_SIZE * (INDEX_HEAD_DIM + 4), + dtype=torch.uint8, + device="cuda", + ) + k = torch.randint( + 1, 127, (seq_len, INDEX_HEAD_DIM), dtype=torch.uint8, device="cuda" + ) + scale = torch.randn(seq_len, dtype=torch.float32, device="cuda") + fp8_dtype = torch.float8_e4m3fnuz if _is_fp8_fnuz else torch.float8_e4m3fn + SetKAndS.triton( + pool, buf, torch.arange(seq_len, device="cuda"), k.view(fp8_dtype), scale + ) + + k_out = torch.zeros_like(k) + scale_out = torch.zeros_like(scale) + gather_index_k_scale_prefix_into( + pool, + buf, + torch.arange(num_pages, dtype=torch.int32, device="cuda"), + seq_len, + k_out, + scale_out, + ) + torch.testing.assert_close(k_out, k, atol=0, rtol=0) + torch.testing.assert_close(scale_out, scale, atol=0, rtol=0) + + +if __name__ == "__main__": + unittest.main()