diff --git a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py index f0c7a99ec5..dbfdefb410 100644 --- a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py +++ b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py @@ -41,8 +41,9 @@ class GetK: def slow( cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor ): - num_pages = (seq_len + pool.page_size - 1) // pool.page_size - seq_len_ = num_pages * pool.page_size + page_size = pool.slots_per_page + num_pages = (seq_len + page_size - 1) // page_size + seq_len_ = num_pages * page_size index_k_fp8 = torch.empty( (seq_len_, pool.index_head_dim), dtype=torch.uint8, @@ -50,9 +51,9 @@ class GetK: ) for i in range(num_pages): page_index = page_indices[i] - index_k_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[ - page_index - ][: pool.page_size * pool.index_head_dim].view(-1, pool.index_head_dim) + index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][ + : page_size * pool.index_head_dim + ].view(-1, pool.index_head_dim) return index_k_fp8[:seq_len] @@ -70,7 +71,7 @@ class GetK: # page_indices: (num_pages,), element := a page index buf_numel_per_page = buf.shape[1] - num_k_bytes_per_page = pool.page_size * pool.index_head_dim + num_k_bytes_per_page = pool.slots_per_page * pool.index_head_dim num_k_bytes_per_token = pool.index_head_dim # buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8 @@ -99,7 +100,7 @@ class GetK: buf=buf, page_indices=page_indices, seq_len=seq_len, - page_size=pool.page_size, + page_size=pool.slots_per_page, index_head_dim=pool.index_head_dim, ) @@ -113,8 +114,9 @@ class GetS: def slow( cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor ): - num_pages = (seq_len + pool.page_size - 1) // pool.page_size - seq_len_ = num_pages * pool.page_size + page_size = pool.slots_per_page + num_pages = (seq_len + page_size - 1) // page_size + seq_len_ = num_pages * page_size assert pool.index_head_dim // pool.quant_block_size == 1 index_k_scale_fp8 = torch.empty( (seq_len_, 4), @@ -123,9 +125,9 @@ class GetS: ) for i in range(num_pages): page_index = page_indices[i] - index_k_scale_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[ - page_index - ][pool.page_size * pool.index_head_dim :].view(-1, 4) + index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][ + page_size * pool.index_head_dim : + ].view(-1, 4) return index_k_scale_fp8[:seq_len] @classmethod @@ -138,9 +140,9 @@ class GetS: """ buf_numel_per_page = buf.shape[1] - num_s_bytes_per_page = buf.shape[1] - pool.page_size * pool.index_head_dim + s_offset_in_page = pool.slots_per_page * pool.index_head_dim + num_s_bytes_per_page = buf.shape[1] - s_offset_in_page num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4 - s_offset_in_page = pool.page_size * pool.index_head_dim flat_buf = buf.flatten() flat_indices = ( @@ -168,7 +170,7 @@ class GetS: buf=buf, page_indices=page_indices, seq_len=seq_len, - page_size=pool.page_size, + page_size=pool.slots_per_page, index_head_dim=pool.index_head_dim, ) @@ -196,7 +198,7 @@ class GetKAndS: ): from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype - page_size = pool.page_size + page_size = pool.slots_per_page index_head_dim = pool.index_head_dim quant_block_size = pool.quant_block_size scale_elems = index_head_dim // quant_block_size @@ -252,7 +254,7 @@ class GetKAndS: seq_lens=seq_len_tensor, seq_len_sum=seq_len_sum, max_seq_len=max_seq_len, - page_size=pool.page_size, + page_size=pool.slots_per_page, index_head_dim=pool.index_head_dim, ) @@ -271,7 +273,7 @@ class SetKAndS: loc=loc, index_k=index_k, index_k_scale=index_k_scale, - page_size=pool.page_size, + page_size=pool.slots_per_page, ) diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 0c756907b8..bfa2fd668e 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -33,6 +33,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: """Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook.""" cfg = resolving_view(server_args) from sglang.srt.configs.model_config import ( + get_dsa_index_kpool, is_deepseek_dsa, unwrap_modelopt_quantization_config, ) @@ -139,8 +140,16 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "needs Triton>=3.5.0 or AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1)." ) else: - overrides["page_size"] = 64 - logger.warning("Setting page size to 64 for DeepSeek DSA.") + index_kpool = get_dsa_index_kpool(hf_config) + if index_kpool > 1 and getattr(cfg, "dcp_size", 1) > 1: + raise ValueError( + "--dcp-size > 1 is not supported for DSA with " + f"index_kpool={index_kpool}: a pooled index key spans " + "positions owned by different DCP ranks." + ) + page_size = 64 * index_kpool + overrides["page_size"] = page_size + logger.warning(f"Setting page size to {page_size} for DeepSeek DSA.") elif get_platform().is_xpu: overrides["page_size"] = 128 logger.warning("Setting page size to 128 for DeepSeek DSA on XPU.") diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py index acf04fcb7b..236a00c59a 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py @@ -68,7 +68,7 @@ class DeepseekSparseAttnBackendKPoolMixin: return dsa_impl def _kpool_slots_per_page(self) -> int: - return getattr(self.token_to_kv_pool, "slots_per_page", self.real_page_size) + return self.token_to_kv_pool.slots_per_page def _build_kpool_paged_mqa_schedule_metadata(self) -> bool: if self.device_sm_major == 9: diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 561107bd77..8e4d29cbe2 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -523,7 +523,7 @@ class IndexerKPool(MultiPlatformOp): ) pool_start_id = first_pos // kpool - page_size = get_token_to_kv_pool().page_size + page_size = attn_metadata.page_size use_returned_compressed = return_compressed and ( not write_cache or pool_start_id == 0 ) @@ -955,7 +955,7 @@ class IndexerKPool(MultiPlatformOp): assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) pool = get_token_to_kv_pool() - page_size = pool.page_size + page_size = pool.slots_per_page # DeepGEMM paged-MQA requires 64-token pages. assert page_size == 64, "only support page size 64" @@ -1241,7 +1241,8 @@ class IndexerKPool(MultiPlatformOp): ) pool_size = self.index_kpool - page_size = get_token_to_kv_pool().page_size + page_size = metadata.attn_metadata.page_size + slots_per_page = get_token_to_kv_pool().slots_per_page token_nums = q_fp8.shape[0] tail_pool = pool_size - 1 topk_result = torch.empty( @@ -1355,7 +1356,9 @@ class IndexerKPool(MultiPlatformOp): token_page_table = block_tables[ i, :num_token_pages ].contiguous() - pool_pages = (curr_pool_start + page_size - 1) // page_size + pool_pages = ( + curr_pool_start + slots_per_page - 1 + ) // slots_per_page pooled_page_table = build_pooled_page_table_64( token_page_table, pool_size )[:pool_pages].contiguous() @@ -1395,7 +1398,9 @@ class IndexerKPool(MultiPlatformOp): token_page_table = block_tables[ i, :num_token_pages ].contiguous() - pool_pages = (pool_seq_len + page_size - 1) // page_size + pool_pages = ( + pool_seq_len + slots_per_page - 1 + ) // slots_per_page pooled_page_table = build_pooled_page_table_64( token_page_table, pool_size )[:pool_pages].contiguous() @@ -1494,7 +1499,7 @@ class IndexerKPool(MultiPlatformOp): assert forward_batch.forward_mode.is_extend_without_speculative() - page_size = get_token_to_kv_pool().page_size + page_size = get_token_to_kv_pool().slots_per_page assert page_size == 64, "only support page size 64" assert len(weights.shape) == 3 weights = weights.squeeze(-1) diff --git a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py index 382a0f1582..3edf5c2556 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py @@ -64,7 +64,7 @@ def build_pooled_page_table_64( idx = torch.arange( 0, page_table_64.shape[-1], pool_size, device=page_table_64.device ) - return page_table_64[..., idx] + return page_table_64[..., idx] // pool_size def gather_index_k_scale_prefix_into( @@ -79,7 +79,7 @@ def gather_index_k_scale_prefix_into( assert page_indices.dtype in (torch.int32, torch.int64) assert k_out.dtype == torch.uint8 assert scale_out.dtype == torch.float32 - assert pool.page_size == BLOCK_SIZE_K + assert pool.slots_per_page == BLOCK_SIZE_K assert k_out.shape[0] >= seq_len assert k_out.shape[1] == INDEX_HEAD_DIM assert scale_out.shape[0] >= seq_len @@ -96,11 +96,11 @@ def gather_index_k_scale_prefix_into( page_indices, k_out, scale_out, - PAGE_SIZE=pool.page_size, + PAGE_SIZE=pool.slots_per_page, BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), HEAD_DIM=INDEX_HEAD_DIM, - S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM, + S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * INDEX_HEAD_DIM, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), ) @@ -159,8 +159,6 @@ def kpool_build_ragged_layout( total_q: int, pool_size: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - # One packed-cache page represents 64 pools, so its source is every - # pool_size-th real token page. device = full_page_table.device n_rag = cu_pages_excl.shape[0] concat_page_table = torch.empty( @@ -224,7 +222,11 @@ def _kpool_build_ragged_layout_kernel( mask=p_mask, other=0, ) - tl.store(concat_page_table_ptr + page_start + p_offs, pages, mask=p_mask) + tl.store( + concat_page_table_ptr + page_start + p_offs, + pages // POOL_SIZE, + mask=p_mask, + ) for q_off in tl.range(0, BLOCK_Q * tl.cdiv(q_count, BLOCK_Q), BLOCK_Q): q_offs = q_off + tl.arange(0, BLOCK_Q) @@ -377,11 +379,14 @@ def _update_kpool_write_plan_kernel( ) # Promote the row term before multiplication; 1M-context strides # overflow int32 around row 2048. - packed_page = tl.load( - real_page_table_ptr - + (b * N).to(tl.int64) * real_page_table_stride_0 - + token_page_row - ).to(tl.int64) + packed_page = ( + tl.load( + real_page_table_ptr + + (b * N).to(tl.int64) * real_page_table_stride_0 + + token_page_row + ).to(tl.int64) + // POOL_SIZE + ) write_loc = packed_page * SLOTS_PER_PAGE + (pool_id % SLOTS_PER_PAGE) tl.store( write_loc_out_ptr + b * write_loc_out_stride_0 + p, @@ -398,10 +403,11 @@ def compute_pooled_write_locs( pool_ids = pool_ids.to(torch.int64) pool_page_group = torch.div(pool_ids, BLOCK_SIZE_K, rounding_mode="floor") token_page_row = pool_page_group * pool_size - packed_page = page_table_64.index_select(0, token_page_row.to(torch.int64)) - return packed_page.to(torch.int64) * BLOCK_SIZE_K + torch.remainder( - pool_ids, BLOCK_SIZE_K + packed_page = ( + page_table_64.index_select(0, token_page_row.to(torch.int64)).to(torch.int64) + // pool_size ) + return packed_page * BLOCK_SIZE_K + torch.remainder(pool_ids, BLOCK_SIZE_K) def history_group_budget_for_topk(topk: int, pool_size: int) -> int: @@ -712,7 +718,7 @@ def kpool_softmax_rotate_write_cache( assert slot_score.dtype in KPOOL_SCORE_DTYPES assert ape.dtype == torch.float32 assert buf.dtype == torch.uint8 - assert pool.page_size == BLOCK_SIZE_K + assert pool.slots_per_page == BLOCK_SIZE_K assert pool.index_head_dim == INDEX_HEAD_DIM assert loc.dtype == torch.int64 assert write_cache or return_compressed @@ -771,12 +777,12 @@ def kpool_softmax_rotate_write_cache( slot_score.stride(0), slot_score.stride(1), ape.stride(0), - PAGE_SIZE=pool.page_size, + PAGE_SIZE=pool.slots_per_page, BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), POOL_SIZE=slot_k.shape[1], HEAD_DIM=slot_k.shape[2], - S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim, + S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim, ROUND_SCALE=round_scale, HAS_WRITE_MASK=has_write_mask, RETURN_COMPRESSED=return_compressed, @@ -816,7 +822,7 @@ def kpool_decode_update_and_maybe_write_cache( assert slot_score.dtype == tail_score.dtype assert ape.dtype == torch.float32 assert buf.dtype == torch.uint8 - assert pool.page_size == BLOCK_SIZE_K + assert pool.slots_per_page == BLOCK_SIZE_K assert pool.index_head_dim == INDEX_HEAD_DIM assert tail_k.is_contiguous() assert tail_score.is_contiguous() @@ -1165,7 +1171,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( + row * block_tables_stride_0 + token_page_row * block_tables_stride_1, ) - loc_page_index = packed_page.to(tl.int64) + loc_page_index = packed_page.to(tl.int64) // POOL_SIZE loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE out_k_offsets = _kpool_cache_k_offsets( loc_page_index, diff --git a/python/sglang/srt/layers/attention/dsa/kpool_plan.py b/python/sglang/srt/layers/attention/dsa/kpool_plan.py index 99370894ee..d367d08962 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_plan.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_plan.py @@ -373,8 +373,9 @@ def _kpool_plan_to_gpu( pool_pool_id_t, slots_per_page, rounding_mode="floor" ) token_page_row = pool_page_group * pool_size - packed_page = full_real_page_table[pool_batch_idx_t, token_page_row].to( - torch.int64 + packed_page = ( + full_real_page_table[pool_batch_idx_t, token_page_row].to(torch.int64) + // pool_size ) pool_write_locs = packed_page * slots_per_page + torch.remainder( pool_pool_id_t, slots_per_page diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 307ad3c7d9..bb05d9d59c 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -369,7 +369,6 @@ class DeepseekSparseAttnBackend( self.forward_metadata: DSAMetadata self.device = model_runner.device assert isinstance(model_runner.page_size, int) - self.real_page_size = model_runner.page_size self.num_splits = ( 1 if get_exec().deterministic.enable_deterministic_inference else 0 ) @@ -381,6 +380,7 @@ class DeepseekSparseAttnBackend( ) self.dsa_index_topk = get_dsa_index_topk(hf_config) self.dsa_index_kpool = get_dsa_index_kpool(hf_config) + self.real_page_size = model_runner.page_size // self.dsa_index_kpool self._init_kpool_metadata_fusion() self.max_context_len = model_runner.model_config.context_len self._memory_saver_adapter = TorchMemorySaverAdapter.create( diff --git a/python/sglang/srt/mem_cache/index_key_cache.py b/python/sglang/srt/mem_cache/index_key_cache.py index a1036472fd..2d8482dac0 100644 --- a/python/sglang/srt/mem_cache/index_key_cache.py +++ b/python/sglang/srt/mem_cache/index_key_cache.py @@ -35,7 +35,7 @@ class IndexKeyCache: pool = self.pool return ( num_pages, - pool.page_size + pool.slots_per_page * (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4), ) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 33f6808fa4..953aa9d70c 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -5146,7 +5146,10 @@ class DSATokenToKVPool(MLATokenToKVPool): self.index_kpool = index_kpool self.index_kpool_compress = index_kpool_compress self.tail_extra_slots = tail_extra_slots - self.slots_per_page = self.page_size + assert self.page_size % index_kpool == 0, ( + f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}" + ) + self.slots_per_page = self.page_size // index_kpool if index_buf_size is None: index_buf_size = size self.index_buf_size = index_buf_size @@ -5160,22 +5163,26 @@ class DSATokenToKVPool(MLATokenToKVPool): ) assert len(self.skip_topk_layers) == layer_num + physical_page_size = self.page_size // index_kpool if _is_hip: if aiter_can_use_preshuffle_paged_mqa(): - assert self.page_size % 16 == 0, ( - f"HIP preshuffle requires page_size to be a multiple of 16, got {self.page_size}" + assert physical_page_size % 16 == 0, ( + f"HIP preshuffle requires page_size to be a multiple of 16, got {physical_page_size}" ) else: - assert self.page_size == 1, ( - f"HIP legacy DSA path requires page_size == 1, got {self.page_size}" + assert physical_page_size == 1, ( + f"HIP legacy DSA path requires page_size == 1, got {physical_page_size}" ) elif is_xpu(): - assert self.page_size in ( + assert physical_page_size in ( 64, 128, - ), f"XPU DSA requires page_size 64 or 128, got {self.page_size}" + ), f"XPU DSA requires page_size 64 or 128, got {physical_page_size}" else: - assert self.page_size == 64 + assert physical_page_size == 64, ( + f"DSA requires 64-token physical pages, got page_size={self.page_size} " + f"with index_kpool={index_kpool}" + ) self.index_key_cache = self._create_index_key_cache() self._init_kpool_compress_tail_buffers( index_kpool=index_kpool, diff --git a/python/sglang/srt/mem_cache/pool_host/dsa.py b/python/sglang/srt/mem_cache/pool_host/dsa.py index 05a733f5bb..7ba4b01616 100644 --- a/python/sglang/srt/mem_cache/pool_host/dsa.py +++ b/python/sglang/srt/mem_cache/pool_host/dsa.py @@ -98,16 +98,21 @@ def make_dsa_indexer_pool_decl( pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER ) -> HostPoolDecl: """Index key buffers riding on the full-KV pages: indices and layout both follow KV.""" + index_page_bytes = pool.slots_per_page * dsa_indexer_bytes_per_token_per_layer( + index_head_dim=pool.index_head_dim, + quant_block_size=pool.quant_block_size, + ) + assert index_page_bytes % pool.page_size == 0, ( + f"index page of {index_page_bytes} bytes does not split over " + f"page_size={pool.page_size} tokens" + ) return HostPoolDecl( pool_name=name, device_pool=pool, indices_from_pool=PoolName.KV, layout_source=PoolName.KV, storage_info=HostPoolStorageInfo( - bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer( - index_head_dim=pool.index_head_dim, - quant_block_size=pool.quant_block_size, - ), + bytes_per_token_per_layer=index_page_bytes // pool.page_size, dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype, ), host_pool_builder=DSAIndexerHostPoolBuilder(), diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index b6a54aad5a..3986089f98 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -21,6 +21,7 @@ from sglang.srt.configs.model_config import ( AttentionArch, dsa_layer_skips_topk, get_dsa_index_head_dim, + get_dsa_index_kpool, get_minimax_sparse_attention_config, get_minimax_sparse_disable_value_layer_ids, get_minimax_sparse_layer_ids, @@ -515,8 +516,9 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): allocate_all_layers: bool = False, ) -> int: index_head_dim = get_dsa_index_head_dim(kvc.model_config.hf_config) - indexer_size_per_token = ( - index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4 + indexer_size_per_token = ceil_div( + index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4, + get_dsa_index_kpool(kvc.model_config.hf_config), ) element_size = torch._utils._element_size( DSATokenToKVPool.index_k_with_scale_buffer_dtype @@ -642,9 +644,10 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): ) if is_deepseek_dsa(model_config.hf_config): index_head_dim = get_dsa_index_head_dim(model_config.hf_config) - index_elements = ( + index_elements = ceil_div( index_head_dim - + index_head_dim // DSATokenToKVPool.quant_block_size * 4 + + index_head_dim // DSATokenToKVPool.quant_block_size * 4, + get_dsa_index_kpool(model_config.hf_config), ) self._full_per_token += index_elements * torch._utils._element_size( DSATokenToKVPool.index_k_with_scale_buffer_dtype diff --git a/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py b/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py index 26ce445333..174b3a9008 100644 --- a/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py +++ b/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py @@ -24,7 +24,7 @@ register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-l @unittest.skipUnless(torch.cuda.is_available(), "Test requires CUDA") class TestDsaKpoolMultiPool(CustomTestCase): POOL_SIZE = 4 - PAGE_SIZE = 64 + PAGE_SIZE = 256 SLOTS_PER_PAGE = 64 NUM_DRAFT_TOKENS = 6 @@ -64,10 +64,10 @@ class TestDsaKpoolMultiPool(CustomTestCase): dtype=torch.int32, device="cuda", ) - real_page_table[:num_draft_tokens, 0] = 2 - real_page_table[:num_draft_tokens, 4] = 3 - real_page_table[num_draft_tokens:, 0] = 5 - real_page_table[num_draft_tokens:, 4] = 6 + real_page_table[:num_draft_tokens, 0] = 2 * self.POOL_SIZE + real_page_table[:num_draft_tokens, 4] = 3 * self.POOL_SIZE + real_page_table[num_draft_tokens:, 0] = 5 * self.POOL_SIZE + real_page_table[num_draft_tokens:, 4] = 6 * self.POOL_SIZE update_kpool_write_plan_cuda_graph( write_start=write_start, diff --git a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py index a48d873dff..7905098816 100644 --- a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py +++ b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py @@ -24,7 +24,7 @@ class TestKpoolPreshuffledCacheLayout(CustomTestCase): """A layout mismatch between the two writers never raises; it silently returns the wrong top-k.""" torch.manual_seed(0) - pool = SimpleNamespace(page_size=PAGE_SIZE) + pool = SimpleNamespace(page_size=4 * PAGE_SIZE, slots_per_page=PAGE_SIZE) num_pages, seq_len = 2, PAGE_SIZE + 5 buf = torch.zeros( num_pages, diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index 23c8ff023b..ca8b05e26e 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -671,6 +671,7 @@ class TestKPoolPerRequestChunking(CustomTestCase): get_seqlens_expanded=lambda: seqlens_expanded, topk_transform_method=TopkTransformMethod.RAGGED, attn_metadata=SimpleNamespace( + page_size=64, topk_indices_offset=torch.arange(token_nums, dtype=torch.int32) + 100, ), ) @@ -689,7 +690,8 @@ class TestKPoolPerRequestChunking(CustomTestCase): index_kpool=self.POOL, ) with _chunking_patches( - rows_per_chunk=rows_per_chunk, kv_pool=SimpleNamespace(page_size=64) + rows_per_chunk=rows_per_chunk, + kv_pool=SimpleNamespace(page_size=256, slots_per_page=64), ): result = IndexerKPool._get_topk_ragged_kpool( backend, diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 299b5f07ba..d04028ab4c 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -268,6 +268,8 @@ class TestDraftSidecarPoolDispatch(CustomTestCase): draft_kv_pool.layer_num = 1 draft_kv_pool.size = 800 draft_kv_pool.index_head_dim = 128 + draft_kv_pool.page_size = 64 + draft_kv_pool.slots_per_page = 64 draft_kv_pool.skip_topk_layers = [False] draft_kv_pool.index_key_cache = SimpleNamespace(buffer=[object()]) draft_host_pool = SimpleNamespace(layer_num=1) @@ -333,6 +335,8 @@ def _dsa_pool_stub(*, layer_num: int, size: int = 4096, shard: tuple | None = No pool.qk_rope_head_dim = 64 pool.kv_cache_dim = 576 pool.index_head_dim = 128 + pool.page_size = 64 + pool.slots_per_page = 64 pool.skip_topk_layers = [False] * layer_num pool.index_key_cache = SimpleNamespace(buffer=[object()] * layer_num) pool.layer_shard_enabled = shard is not None diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index e37e445e01..88c66e2fba 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -483,6 +483,8 @@ class TestDSAIndexerPoolDecl(CustomTestCase): qk_rope_head_dim=64, index_head_dim=128, quant_block_size=128, + page_size=64, + slots_per_page=64, skip_topk_layers=[False] * 5, ) diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index aa94fde703..0862247e11 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -83,6 +83,8 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): end_layer=2, index_head_dim=8, quant_block_size=4, + page_size=2, + slots_per_page=2, skip_topk_layers=[False] * 2, ) indexer_host = DSAIndexerPoolHost(