diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 2635f2b698..91f279ea13 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import torch @@ -24,7 +25,10 @@ from sglang.srt.layers.attention.dsa.utils import ( from sglang.srt.layers.attention.mqa_logits_utils import ( MQA_LOGITS_BYTES_PER_ELEM, MQA_LOGITS_MAX_BYTES_ROCM, + mqa_logits_budget_bytes, + mqa_logits_row_bytes, mqa_logits_rows_per_chunk, + mqa_logits_should_chunk, ) from sglang.srt.layers.layernorm import LayerNorm from sglang.srt.layers.utils import MultiPlatformOp @@ -57,8 +61,11 @@ from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( is_in_breakable_cuda_graph, ) +from sglang.srt.model_executor.runner_utils import capture_mode from sglang.srt.runtime_context import get_device, get_exec +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool @@ -69,6 +76,56 @@ def _should_fuse_kpool_topk(metadata: BaseIndexerMetadata) -> bool: ) +def _slice_rows( + *, tensor: Optional[torch.Tensor], rows: slice +) -> Optional[torch.Tensor]: + return None if tensor is None else tensor[rows] + + +def _mqa_logits_row_chunks( + *, num_rows: int, num_cols: int, device: torch.device +) -> Tuple[slice, ...]: + # Real capture records a fixed launch count; breakable-graph replay also sets + # get_is_capture_mode(), but its eager breaks run here and need the budget. + if capture_mode.is_capture_mode or torch.cuda.is_current_stream_capturing(): + return (slice(0, num_rows),) + device_index = device.index + assert device_index is not None, "q_fp8 must be on an indexed CUDA device" + # The logits kernel (DeepGEMM, or AITER on ROCm) allocates them outside every + # pool sized by mem_fraction_static, so only the free-memory budget bounds them. + need_chunk, budget_bytes = mqa_logits_should_chunk( + num_rows=num_rows, + num_cols=num_cols, + get_budget_bytes=lambda: mqa_logits_budget_bytes( + device_index=device_index, allow_sync=True + ), + rocm=is_hip(), + ) + rows_per_chunk = ( + mqa_logits_rows_per_chunk( + num_rows=num_rows, + row_bytes=mqa_logits_row_bytes(num_cols), + budget_bytes=budget_bytes, + ) + if need_chunk + else None + ) + if rows_per_chunk is None: + return (slice(0, num_rows),) + logger.debug( + "kpool indexer chunks %d query rows x %d pooled cols into %d-row " + "chunks (logits budget %d bytes)", + num_rows, + num_cols, + rows_per_chunk, + budget_bytes, + ) + return tuple( + slice(start, min(start + rows_per_chunk, num_rows)) + for start in range(0, num_rows, rows_per_chunk) + ) + + class IndexerKPool(MultiPlatformOp): def __init__( self, @@ -1007,18 +1064,81 @@ class IndexerKPool(MultiPlatformOp): ) return topk_result - def _should_chunk_mqa_logits( - self, num_q: int, num_k: int, device: torch.device - ) -> Tuple[bool, int]: - if num_q * num_k < 8_000_000: - return False, 0 - - free_mem, total_mem = torch.cuda.mem_get_info(device) - bytes_per_elem = 4 - logits_bytes = num_q * num_k * bytes_per_elem - - need_chunk = (logits_bytes * 2 > free_mem) or (logits_bytes > total_mem * 0.3) - return need_chunk, free_mem + def _kpool_topk_by_row_chunks( + self, + *, + q_fp8: torch.Tensor, + weights: torch.Tensor, + kv_fp8: Optional[Tuple[torch.Tensor, torch.Tensor]], + logits_starts: Optional[torch.Tensor], + logits_ends: torch.Tensor, + pool_lens: torch.Tensor, + seq_lens: Optional[torch.Tensor], + page_table: Optional[torch.Tensor], + page_table_row_index: Optional[torch.Tensor], + topk_offsets: Optional[torch.Tensor], + topk_row_starts: Optional[torch.Tensor], + row_chunks: Tuple[slice, ...], + out_rows: Optional[int], + ) -> torch.Tensor: + # Each row's top-k reads only its own logits row, pooled length and + # page-table row, so row chunks select the same pages as one pass. + single_chunk = len(row_chunks) == 1 + topk_result = None + for rows in row_chunks: + if kv_fp8 is None: + # No pooled keys yet: every row scores an empty key set. + logits = torch.empty( + (rows.stop - rows.start, 0), + dtype=torch.float32, + device=q_fp8.device, + ) + else: + assert logits_starts is not None + k_fp8, k_scale = kv_fp8 + logits = self._fp8_mqa_logits( + q_fp8=q_fp8[rows].contiguous(), + k_fp8=k_fp8, + k_scale=k_scale, + weights=weights[rows].contiguous(), + starts=logits_starts[rows], + ends=logits_ends[rows], + clean_logits=True, + ) + topk_chunk = self._topk_from_kpool_logits( + logits=logits, + pool_lens=pool_lens[rows], + seq_lens=_slice_rows(tensor=seq_lens, rows=rows), + # Addressed by request-pool ID through page_table_row_index, + # so never sliced; without a row index it is per-query. + page_table=( + page_table + if page_table_row_index is not None + else _slice_rows(tensor=page_table, rows=rows) + ), + topk_offsets=_slice_rows(tensor=topk_offsets, rows=rows), + row_starts=_slice_rows(tensor=topk_row_starts, rows=rows), + out_rows=out_rows if single_chunk else None, + page_table_row_index=_slice_rows( + tensor=page_table_row_index, rows=rows + ), + ) + if single_chunk: + return topk_chunk + # Free this chunk's logits before the next one allocates. + del logits + if topk_result is None: + # Rows past the last chunk are padding, matching out_rows. + num_out_rows = row_chunks[-1].stop if out_rows is None else out_rows + topk_result = torch.full( + (num_out_rows, topk_chunk.shape[1]), + -1, + dtype=topk_chunk.dtype, + device=topk_chunk.device, + ) + topk_result[rows] = topk_chunk + assert topk_result is not None + return topk_result def _get_topk_ragged_kpool_plan( self, @@ -1050,6 +1170,7 @@ class IndexerKPool(MultiPlatformOp): f"plan has more real rows ({n_real}) than q_fp8 ({total_q})" ) + kv_fp8 = None if total_k_rows > 0: k_u8 = plan.ragged_k_u8 k_scale = plan.ragged_k_scale @@ -1064,17 +1185,7 @@ class IndexerKPool(MultiPlatformOp): scale_out=k_scale, ) k_fp8 = k_u8.view(torch.float8_e4m3fn) - logits = self._fp8_mqa_logits( - q_fp8[:n_real].contiguous(), - k_fp8.contiguous(), - k_scale.contiguous(), - weights[:n_real].contiguous(), - ks_per_q, - ke_per_q, - clean_logits=True, - ) - else: - logits = torch.empty((n_real, 0), dtype=torch.float32, device=device) + kv_fp8 = (k_fp8.contiguous(), k_scale.contiguous()) topk_method = metadata.topk_transform_method attn_metadata = metadata.attn_metadata @@ -1087,16 +1198,25 @@ class IndexerKPool(MultiPlatformOp): page_table_row_index_all = plan.ragged_paged_page_table_row_index elif topk_method == TopkTransformMethod.RAGGED: topk_offsets_all = attn_metadata.topk_indices_offset + # The plan's table is req_to_token, indexed by request-pool ID. + assert page_table_all is None or page_table_row_index_all is not None - return self._topk_from_kpool_logits( - logits, - pool_lens, + return self._kpool_topk_by_row_chunks( + q_fp8=q_fp8, + weights=weights, + kv_fp8=kv_fp8, + logits_starts=ks_per_q, + logits_ends=ke_per_q, + pool_lens=pool_lens, seq_lens=seq_lens_expanded, page_table=page_table_all, - topk_offsets=topk_offsets_all, - row_starts=ks_per_q, - out_rows=total_q, page_table_row_index=page_table_row_index_all, + topk_offsets=topk_offsets_all, + topk_row_starts=ks_per_q, + row_chunks=_mqa_logits_row_chunks( + num_rows=n_real, num_cols=total_k_rows, device=device + ), + out_rows=total_q, ) def _get_topk_ragged_kpool( @@ -1297,19 +1417,10 @@ class IndexerKPool(MultiPlatformOp): and zero_starts_by_batch[i] is not None else torch.zeros((q_len,), dtype=torch.int32, device=q_fp8.device) ) - local_logits = self._fp8_mqa_logits( - q_fp8[q_slice].contiguous(), - k_fp8.contiguous(), - k_scale.contiguous(), - weights[q_slice].contiguous(), - row_starts, - local_pool_lens, - clean_logits=True, - ) + local_kv_fp8 = (k_fp8.contiguous(), k_scale.contiguous()) else: - local_logits = torch.empty( - (q_len, 0), dtype=torch.float32, device=q_fp8.device - ) + row_starts = None + local_kv_fp8 = None page_table_local = None topk_offsets_local = None @@ -1330,12 +1441,23 @@ class IndexerKPool(MultiPlatformOp): ): topk_offsets_local = topk_offsets[q_slice] - local_topk = self._topk_from_kpool_logits( - local_logits, - local_pool_lens, + # One request's logits still scale with its own context. + local_topk = self._kpool_topk_by_row_chunks( + q_fp8=q_fp8[q_slice], + weights=weights[q_slice], + kv_fp8=local_kv_fp8, + logits_starts=row_starts, + logits_ends=local_pool_lens, + pool_lens=local_pool_lens, seq_lens=local_seqlens, page_table=page_table_local, + page_table_row_index=None, topk_offsets=topk_offsets_local, + topk_row_starts=None, + row_chunks=_mqa_logits_row_chunks( + num_rows=q_len, num_cols=pool_seq_len, device=q_fp8.device + ), + out_rows=None, ) topk_result[q_slice] = local_topk diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index f43c5e6a4b..0ae4e845ac 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -1,8 +1,10 @@ -"""CPU scheduling checks: stream dependencies, cache contracts and gate math.""" +"""CPU kpool indexer checks: streams, cache contracts, gate math, logits chunking.""" +import sys import unittest -from contextlib import contextmanager, nullcontext -from types import MethodType, SimpleNamespace +from contextlib import ExitStack, contextmanager, nullcontext +from functools import partial +from types import MethodType, ModuleType, SimpleNamespace from unittest.mock import Mock, patch import torch @@ -11,8 +13,11 @@ import torch.nn.functional as F from sglang.kernels.ops.attention.dsa import triton_kernel from sglang.srt.layers.attention.dsa import dsa_indexer_kpool as indexer_module from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool +from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -312,5 +317,415 @@ class TestKPoolStreamScheduling(unittest.TestCase): self.assertNotIn(("topk", "main"), streams.trace) +# Over the 8M-element skip threshold, so the budget decides. +_LARGE_ROWS, _LARGE_COLS = 16384, 229376 + + +def _plan_chunks(*, num_rows, num_cols, budget_bytes, capture_mode=False): + with ( + patch.object( + indexer_module, "mqa_logits_budget_bytes", return_value=budget_bytes + ) as budget, + patch.object(indexer_module, "is_hip", return_value=False), + patch.object(torch.cuda, "is_current_stream_capturing", return_value=False), + patch.object(indexer_module.capture_mode, "is_capture_mode", capture_mode), + ): + chunks = indexer_module._mqa_logits_row_chunks( + num_rows=num_rows, num_cols=num_cols, device=torch.device("cuda", 0) + ) + return chunks, budget + + +class TestKPoolMqaLogitsRowChunks(CustomTestCase): + def test_small_matrices_skip_the_budget_query(self): + chunks, budget = _plan_chunks(num_rows=64, num_cols=1024, budget_bytes=1 << 20) + self.assertEqual(chunks, (slice(0, 64),)) + budget.assert_not_called() + + def test_chunks_cover_every_row_once_and_fit_the_budget(self): + budget_bytes = 6 << 30 + chunks, _ = _plan_chunks( + num_rows=_LARGE_ROWS, num_cols=_LARGE_COLS, budget_bytes=budget_bytes + ) + self.assertGreater(len(chunks), 1) + self.assertEqual(chunks[0].start, 0) + self.assertEqual(chunks[-1].stop, _LARGE_ROWS) + for prev, nxt in zip(chunks, chunks[1:]): + self.assertEqual(prev.stop, nxt.start) + for rows in chunks: + self.assertGreater(rows.stop, rows.start) + self.assertLessEqual( + (rows.stop - rows.start) * _LARGE_COLS * 4, budget_bytes + ) + + def test_a_matrix_within_budget_runs_in_one_call(self): + chunks, _ = _plan_chunks( + num_rows=_LARGE_ROWS, num_cols=_LARGE_COLS, budget_bytes=64 << 30 + ) + self.assertEqual(chunks, (slice(0, _LARGE_ROWS),)) + + def test_real_capture_keeps_one_call_without_a_budget_query(self): + chunks, budget = _plan_chunks( + num_rows=_LARGE_ROWS, + num_cols=_LARGE_COLS, + budget_bytes=6 << 30, + capture_mode=True, + ) + self.assertEqual(chunks, (slice(0, _LARGE_ROWS),)) + budget.assert_not_called() + + def test_breakable_graph_replay_still_chunks(self): + """Breakable-graph replay sets get_is_capture_mode(), but its eager breaks + run this path for real and must stay budgeted.""" + with patch.object( + indexer_module.capture_mode, "is_in_breakable_cuda_graph", return_value=True + ): + self.assertTrue(get_is_capture_mode()) + chunks, _ = _plan_chunks( + num_rows=_LARGE_ROWS, num_cols=_LARGE_COLS, budget_bytes=6 << 30 + ) + self.assertGreater(len(chunks), 1) + + +def _row_chunks(*, rows_per_chunk, num_rows, num_cols, device): + if rows_per_chunk is None: + return (slice(0, num_rows),) + return tuple( + slice(start, min(start + rows_per_chunk, num_rows)) + for start in range(0, num_rows, rows_per_chunk) + ) + + +def _fake_deep_gemm(): + # Logits carry each row's weight, so a mis-offset q/weights slice shows up. + return SimpleNamespace( + fp8_mqa_logits=lambda q, kv, w, ks, ke, clean_logits: ( + w[:, :1].expand(q.shape[0], kv[0].shape[0]).clone() + ) + ) + + +def _encoding_topk(*, width, calls): + def topk( + *, + logits, + pool_lens, + seq_lens, + page_table, + topk_offsets, + row_starts, + out_rows, + page_table_row_index, + ): + calls.append( + {"page_table": page_table, "page_table_row_index": page_table_row_index} + ) + # Each row encodes the weight it was scored with, its pooled length and + # its request or offset index. + if page_table_row_index is not None: + index = page_table_row_index + elif topk_offsets is not None: + index = topk_offsets + else: + index = torch.zeros_like(pool_lens) + row_ids = ( + logits[:, 0].to(torch.int32) * 1_000_000 + + pool_lens.to(torch.int32) * 1_000 + + index.to(torch.int32) + ) + result = row_ids.unsqueeze(1).expand(-1, width).contiguous() + # topk_from_pooled_history_logits pads to out_rows with -1. + if out_rows is not None and out_rows > result.shape[0]: + padded = torch.full((out_rows, width), -1, dtype=torch.int32) + padded[: result.shape[0]] = result + result = padded + return result + + return topk + + +def _expected_rows(*, pool_lens, index, width, pad_rows=0): + # Row r scored with its own inputs: weight r + 1 (see _fake_deep_gemm). + weight_ids = torch.arange(1, pool_lens.shape[0] + 1, dtype=torch.int32) + row_ids = weight_ids * 1_000_000 + pool_lens * 1_000 + index + rows = row_ids.to(torch.int32).unsqueeze(1).expand(-1, width) + padding = torch.full((pad_rows, width), -1, dtype=torch.int32) + return torch.cat([rows, padding]) + + +def _chunk_backend(*, topk, index_topk=4, index_kpool=4): + backend = SimpleNamespace( + index_topk=index_topk, + index_kpool=index_kpool, + alt_stream=None, + _topk_from_kpool_logits=topk, + _get_index_k_read_buffer=lambda pool, layer_id: None, + _fp8_mqa_logits=IndexerKPool._fp8_mqa_logits, + ) + backend._kpool_topk_by_row_chunks = MethodType( + IndexerKPool._kpool_topk_by_row_chunks, backend + ) + return backend + + +def _fp8_zeros(*shape): + return torch.zeros(shape, dtype=torch.uint8).view(torch.float8_e4m3fn) + + +def _fake_aiter_modules(): + # _fp8_mqa_logits imports AITER lazily on ROCm; same row encoding as DeepGEMM. + kernel = ModuleType("aiter.ops.triton.fp8_mqa_logits") + kernel.fp8_mqa_logits = lambda q, k, scale, w, starts, ends, clean_logits: ( + w[:, :1].expand(q.shape[0], k.shape[0]).clone() + ) + return { + "aiter": ModuleType("aiter"), + "aiter.ops": ModuleType("aiter.ops"), + "aiter.ops.triton": ModuleType("aiter.ops.triton"), + "aiter.ops.triton.fp8_mqa_logits": kernel, + } + + +def _refuse_deep_gemm(*args, **kwargs): + raise AssertionError("ROCm reached DeepGEMM instead of AITER") + + +@contextmanager +def _chunking_patches(*, rows_per_chunk, kv_pool, rocm=False): + deep_gemm = ( + SimpleNamespace(fp8_mqa_logits=_refuse_deep_gemm) if rocm else _fake_deep_gemm() + ) + with ExitStack() as stack: + # deep_gemm is bound only under `if is_cuda()`, so create it on CPU. + stack.enter_context( + patch.object(indexer_module, "deep_gemm", deep_gemm, create=True) + ) + stack.enter_context( + patch.object( + indexer_module, + "_mqa_logits_row_chunks", + partial(_row_chunks, rows_per_chunk=rows_per_chunk), + ) + ) + stack.enter_context( + patch.object(indexer_module, "get_token_to_kv_pool", return_value=kv_pool) + ) + stack.enter_context( + patch.object(indexer_module, "_should_fuse_kpool_topk", return_value=True) + ) + stack.enter_context(patch.object(indexer_module, "is_hip", return_value=rocm)) + if rocm: + stack.enter_context(patch.dict(sys.modules, _fake_aiter_modules())) + yield + + +class TestKPoolPlanChunking(CustomTestCase): + NUM_REQ_SLOTS = 64 + TOPK = 4 + PAD_ROWS = 5 + + def _drive(self, *, req_ids, rows_per_chunk, topk_method, rocm=False): + n_real = len(req_ids) + total_k_rows = 16 + page_table = torch.arange(self.NUM_REQ_SLOTS * 8, dtype=torch.int32).reshape( + self.NUM_REQ_SLOTS, 8 + ) + plan = SimpleNamespace( + seq_lens_expanded=torch.full((n_real,), 8, dtype=torch.int32), + pooled_seq_lens_expanded=torch.arange(1, n_real + 1, dtype=torch.int32), + ragged_q_ks=torch.zeros(n_real, dtype=torch.int32), + ragged_q_ke=torch.full((n_real,), total_k_rows, dtype=torch.int32), + ragged_total_k_rows=total_k_rows, + ragged_k_u8=torch.zeros((total_k_rows, 4), dtype=torch.uint8), + ragged_k_scale=torch.zeros((total_k_rows, 1), dtype=torch.float32), + ragged_concat_page_table=torch.zeros(total_k_rows, dtype=torch.int32), + ragged_paged_page_table=page_table, + ragged_paged_page_table_row_index=torch.tensor(req_ids, dtype=torch.int32), + ) + # Padded batch: q_fp8 has rows past the plan's real ones. + total_q = n_real + self.PAD_ROWS + metadata = SimpleNamespace( + attn_metadata=SimpleNamespace( + kpool_extend_plan=plan, + topk_indices_offset=torch.arange(total_q, dtype=torch.int32) + 100, + ), + topk_transform_method=topk_method, + ) + weights = torch.arange(1, total_q + 1, dtype=torch.float32).reshape( + total_q, 1, 1 + ) + calls = [] + backend = _chunk_backend(topk=_encoding_topk(width=self.TOPK, calls=calls)) + with ( + _chunking_patches( + rows_per_chunk=rows_per_chunk, kv_pool=object(), rocm=rocm + ), + patch( + "sglang.srt.layers.attention.dsa.kpool_fp8_index." + "gather_index_k_scale_prefix_into", + lambda **kwargs: None, + ), + ): + result = IndexerKPool._get_topk_ragged_kpool_plan( + backend, + forward_batch=None, + layer_id=0, + q_fp8=_fp8_zeros(total_q, 4), + weights=weights, + metadata=metadata, + ) + return result, calls, page_table + + def test_every_chunk_gets_the_whole_request_indexed_table(self): + """Every chunk must see req_to_token whole: a sliced table resolves + request 1 to row 17 once a 16-row chunk shifts its base.""" + req_ids = [1, 9, 2, 40, 7, 3, 63, 0] * 4 + _, calls, page_table = self._drive( + req_ids=req_ids, rows_per_chunk=16, topk_method=TopkTransformMethod.PAGED + ) + self.assertEqual(len(calls), 2) + for chunk_idx, call in enumerate(calls): + self.assertIs(call["page_table"], page_table) + start = chunk_idx * 16 + torch.testing.assert_close( + call["page_table_row_index"], + torch.tensor(req_ids[start : start + 16], dtype=torch.int32), + ) + + def _reference(self, *, req_ids, topk_method): + # Every real row scored with its own inputs, then -1 padding. + n_real = len(req_ids) + index = ( + torch.tensor(req_ids, dtype=torch.int32) + if topk_method == TopkTransformMethod.PAGED + else torch.arange(n_real, dtype=torch.int32) + 100 + ) + return _expected_rows( + pool_lens=torch.arange(1, n_real + 1, dtype=torch.int32), + index=index, + width=self.TOPK, + pad_rows=self.PAD_ROWS, + ) + + def test_chunked_rows_match_the_single_call_including_padding(self): + req_ids = [5, 11, 2, 60, 9, 33, 1] * 4 + for method in (TopkTransformMethod.PAGED, TopkTransformMethod.RAGGED): + with self.subTest(method=method): + expected, one_call, _ = self._drive( + req_ids=req_ids, rows_per_chunk=None, topk_method=method + ) + actual, chunked, _ = self._drive( + req_ids=req_ids, rows_per_chunk=5, topk_method=method + ) + self.assertEqual(len(one_call), 1) + self.assertGreater(len(chunked), 1) + reference = self._reference(req_ids=req_ids, topk_method=method) + torch.testing.assert_close(expected, reference, rtol=0, atol=0) + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + + def test_chunked_rows_score_through_aiter_on_rocm(self): + """On ROCm every row chunk must score through _fp8_mqa_logits, which + routes to AITER; reaching DeepGEMM there fails the forward.""" + req_ids = [5, 11, 2, 60, 9, 33, 1] * 4 + actual, chunked, _ = self._drive( + req_ids=req_ids, + rows_per_chunk=5, + topk_method=TopkTransformMethod.PAGED, + rocm=True, + ) + self.assertGreater(len(chunked), 1) + torch.testing.assert_close( + actual, + self._reference(req_ids=req_ids, topk_method=TopkTransformMethod.PAGED), + rtol=0, + atol=0, + ) + + +def _causal_seq_lens(*, q_lens, seq_lens): + # Extend row j of a request attends to its prefix plus rows 0..j. + return torch.cat( + [ + torch.arange(seq_len - q_len + 1, seq_len + 1, dtype=torch.int32) + for q_len, seq_len in zip(q_lens, seq_lens) + ] + ) + + +class TestKPoolPerRequestChunking(CustomTestCase): + TOPK = 4 + POOL = 4 + + def _drive(self, *, q_lens, seq_lens, rows_per_chunk): + token_nums = sum(q_lens) + forward_batch = SimpleNamespace( + batch_size=len(q_lens), + extend_seq_lens_cpu=list(q_lens), + seq_lens_cpu=torch.tensor(seq_lens, dtype=torch.int64), + req_pool_indices=torch.tensor([7, 3][: len(q_lens)], dtype=torch.int64), + ) + seqlens_expanded = _causal_seq_lens(q_lens=q_lens, seq_lens=seq_lens) + metadata = SimpleNamespace( + get_page_table_64=lambda: torch.zeros((len(q_lens), 1), dtype=torch.int32), + get_seqlens_expanded=lambda: seqlens_expanded, + topk_transform_method=TopkTransformMethod.RAGGED, + attn_metadata=SimpleNamespace( + topk_indices_offset=torch.arange(token_nums, dtype=torch.int32) + 100, + ), + ) + # Fully cached current K: the path scores it without a pool gather. + cache = [ + (0, _fp8_zeros(seq_len // self.POOL, 4), torch.zeros(seq_len // self.POOL)) + for seq_len in seq_lens + ] + weights = torch.arange(1, token_nums + 1, dtype=torch.float32).reshape( + token_nums, 1 + ) + calls = [] + backend = _chunk_backend( + topk=_encoding_topk(width=self.TOPK + self.POOL - 1, calls=calls), + index_topk=self.TOPK, + index_kpool=self.POOL, + ) + with _chunking_patches( + rows_per_chunk=rows_per_chunk, kv_pool=SimpleNamespace(page_size=64) + ): + result = IndexerKPool._get_topk_ragged_kpool( + backend, + forward_batch=forward_batch, + layer_id=0, + q_fp8=_fp8_zeros(token_nums, 4), + weights=weights, + metadata=metadata, + extend_pooled_cache=cache, + ) + return result, calls + + def test_a_long_request_after_another_matches_the_single_call(self): + """A request's row chunks slice relative to that request, not to the + batch; the second request here starts at q offset 5.""" + q_lens, seq_lens = [5, 19], [40, 96] + expected, one_call = self._drive( + q_lens=q_lens, seq_lens=seq_lens, rows_per_chunk=None + ) + actual, chunked = self._drive( + q_lens=q_lens, seq_lens=seq_lens, rows_per_chunk=4 + ) + self.assertEqual(len(one_call), 2) + self.assertGreater(len(chunked), len(one_call)) + pool_lens = torch.div( + _causal_seq_lens(q_lens=q_lens, seq_lens=seq_lens), + self.POOL, + rounding_mode="floor", + ) + reference = _expected_rows( + pool_lens=pool_lens, + index=torch.arange(pool_lens.shape[0], dtype=torch.int32) + 100, + width=self.TOPK + self.POOL - 1, + ) + torch.testing.assert_close(expected, reference, rtol=0, atol=0) + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + + if __name__ == "__main__": unittest.main()