From 9e26600f3a02d1f11e3a5eba01ccb5cd58d5eeeb Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Tue, 6 Oct 2026 21:05:29 -0700 Subject: [PATCH] [DSA] Clarify pooled-page naming and ownership (#42671) --- .../ops/attention/dsa/index_buf_accessor.py | 18 +- .../ops/attention/dsa/kpool_topk_transform.py | 10 +- .../attention/dsa_kpool_metadata/decode.py | 18 +- .../dsa_kpool_metadata/draft_extend.py | 14 +- .../attention/dsa_kpool_metadata/verify.py | 18 +- .../kernels/ops/attention/dsa_metadata.py | 46 ++--- .../arg_groups/model_overrides/deepseek_v2.py | 4 +- .../layers/attention/dsa/dsa_backend_kpool.py | 78 ++++---- .../dsa/dsa_backend_mtp_precompute.py | 20 +- .../layers/attention/dsa/dsa_indexer_kpool.py | 32 ++-- .../attention/dsa/dsa_metadata_manager.py | 9 +- .../layers/attention/dsa/kpool_fp8_index.py | 178 +++++++++-------- .../srt/layers/attention/dsa/kpool_plan.py | 180 +++++++++--------- .../srt/layers/attention/dsa_backend.py | 43 +++-- .../sglang/srt/mem_cache/index_key_cache.py | 2 +- python/sglang/srt/mem_cache/memory_pool.py | 6 +- python/sglang/srt/mem_cache/pool_host/dsa.py | 2 +- python/sglang/test/kits/dsa_metadata_kit.py | 4 +- .../kernels/ops/attention/test_dsa_indexer.py | 2 +- .../test_dsa_kpool_metadata_fusion.py | 8 +- .../attention/test_dsa_kpool_multi_pool.py | 18 +- .../ops/attention/test_dsa_metadata.py | 91 ++++----- .../attention/test_dsa_multi_ctas_counter.py | 2 +- .../test_kpool_topk_transform_fused.py | 36 +++- .../attention/test_dsa_backend_dp_padding.py | 2 +- .../test_kpool_preshuffled_cache_layout.py | 2 +- .../layers/test_kpool_stream_scheduling.py | 2 +- .../mem_cache/test_hybrid_pool_assembler.py | 4 +- .../unit/mem_cache/test_mem_pool_host.py | 2 +- .../test_mla_host_dedup_primitives.py | 2 +- 30 files changed, 436 insertions(+), 417 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py index dbfdefb410..ed40d0b85b 100644 --- a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py +++ b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py @@ -41,7 +41,7 @@ class GetK: def slow( cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor ): - page_size = pool.slots_per_page + page_size = pool.index_page_size num_pages = (seq_len + page_size - 1) // page_size seq_len_ = num_pages * page_size index_k_fp8 = torch.empty( @@ -71,7 +71,7 @@ class GetK: # page_indices: (num_pages,), element := a page index buf_numel_per_page = buf.shape[1] - num_k_bytes_per_page = pool.slots_per_page * pool.index_head_dim + num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim num_k_bytes_per_token = pool.index_head_dim # buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8 @@ -100,7 +100,7 @@ class GetK: buf=buf, page_indices=page_indices, seq_len=seq_len, - page_size=pool.slots_per_page, + page_size=pool.index_page_size, index_head_dim=pool.index_head_dim, ) @@ -114,7 +114,7 @@ class GetS: def slow( cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor ): - page_size = pool.slots_per_page + page_size = pool.index_page_size num_pages = (seq_len + page_size - 1) // page_size seq_len_ = num_pages * page_size assert pool.index_head_dim // pool.quant_block_size == 1 @@ -140,7 +140,7 @@ class GetS: """ buf_numel_per_page = buf.shape[1] - s_offset_in_page = pool.slots_per_page * pool.index_head_dim + s_offset_in_page = pool.index_page_size * pool.index_head_dim num_s_bytes_per_page = buf.shape[1] - s_offset_in_page num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4 @@ -170,7 +170,7 @@ class GetS: buf=buf, page_indices=page_indices, seq_len=seq_len, - page_size=pool.slots_per_page, + page_size=pool.index_page_size, index_head_dim=pool.index_head_dim, ) @@ -198,7 +198,7 @@ class GetKAndS: ): from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype - page_size = pool.slots_per_page + page_size = pool.index_page_size index_head_dim = pool.index_head_dim quant_block_size = pool.quant_block_size scale_elems = index_head_dim // quant_block_size @@ -254,7 +254,7 @@ class GetKAndS: seq_lens=seq_len_tensor, seq_len_sum=seq_len_sum, max_seq_len=max_seq_len, - page_size=pool.slots_per_page, + page_size=pool.index_page_size, index_head_dim=pool.index_head_dim, ) @@ -273,7 +273,7 @@ class SetKAndS: loc=loc, index_k=index_k, index_k_scale=index_k_scale, - page_size=pool.slots_per_page, + page_size=pool.index_page_size, ) diff --git a/python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py b/python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py index 89f066f082..9622cc3a8e 100644 --- a/python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py +++ b/python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py @@ -29,7 +29,7 @@ def _jit_kpool_topk_transform_module(group_topk: int) -> Module: def fast_kpool_topk_transform_fused( score: torch.Tensor, lengths: torch.Tensor, - pool_size: int, + kpool: int, topk: int, page_table: Optional[torch.Tensor] = None, topk_indices_offset: Optional[torch.Tensor] = None, @@ -37,8 +37,8 @@ def fast_kpool_topk_transform_fused( seq_lens: Optional[torch.Tensor] = None, page_table_row_index: Optional[torch.Tensor] = None, ) -> torch.Tensor: - assert topk % pool_size == 0 - group_topk = topk // pool_size + assert topk % kpool == 0 + group_topk = topk // kpool assert group_topk in SUPPORTED_GROUP_TOPK, ( "fast_kpool_topk_transform supports pool-level topk " f"{SUPPORTED_GROUP_TOPK}, got {group_topk}" @@ -53,7 +53,7 @@ def fast_kpool_topk_transform_fused( assert page_table_row_index.dim() == 1 assert page_table_row_index.shape[0] == score.shape[0] - out_cols = topk + (pool_size - 1 if seq_lens is not None else 0) + out_cols = topk + (kpool - 1 if seq_lens is not None else 0) dst_token_indices = score.new_empty((score.shape[0], out_cols), dtype=torch.int32) module = _jit_kpool_topk_transform_module(group_topk) @@ -61,7 +61,7 @@ def fast_kpool_topk_transform_fused( score, lengths, dst_token_indices, - pool_size, + kpool, page_table, topk_indices_offset, row_starts, diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py index 0263dad488..f9ff0a7ca0 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py +++ b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py @@ -40,7 +40,7 @@ def _fused_dsa_decode_metadata_kernel( num_splits, dsa_index_topk: tl.constexpr, index_kpool: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGE_TABLE_1: tl.constexpr, BLOCK_BS: tl.constexpr, @@ -111,13 +111,13 @@ def _fused_dsa_decode_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + row_i64 * real_page_table_stride_0 + real_cols * real_page_table_stride_1, - vals // real_page_size, + vals // physical_page_size, mask=real_mask, ) @@ -135,16 +135,16 @@ def fused_dsa_decode_metadata( bs: int, max_len: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, index_kpool: int = 1, ) -> None: """Fill decode-graph DSA metadata (seqlens + page tables) from req_to_token. ``page_table_1`` (the wide page_size=1 table) is optional: pass ``None`` to skip materializing it and write only the compact ``real_page_table`` - (page_size=``real_page_size``). This is used by the fused decode CUDA graph, + (page_size=``physical_page_size``). This is used by the fused decode CUDA graph, where the wide table is never read (attention uses topk_indices, the indexer - uses real_page_table); ``real_page_size`` must be >1 in that case. When a + uses real_page_table); ``physical_page_size`` must be >1 in that case. When a tensor is passed, behavior is unchanged (both tables are written). Contract: each page-table row is written only over its live prefix @@ -170,7 +170,7 @@ def fused_dsa_decode_metadata( return assert index_kpool > 0 - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -217,7 +217,7 @@ def fused_dsa_decode_metadata( num_splits, dsa_index_topk, index_kpool, - real_page_size, + physical_page_size, has_real_page_table, has_page_table_1, BLOCK_BS=block_bs, diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py index 8dc75674c8..c60e9e8c13 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py +++ b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py @@ -45,7 +45,7 @@ def _fused_dsa_draft_extend_metadata_kernel( num_splits, dsa_index_topk: tl.constexpr, index_kpool: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGE_TABLE_1: tl.constexpr, STATIC_EXTEND_LEN: tl.constexpr, @@ -183,13 +183,13 @@ def _fused_dsa_draft_extend_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n[None, :] % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n[None, :] % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + out_rows_i64[:, None] * real_page_table_stride_0 + real_cols[None, :] * real_page_table_stride_1, - (vals // real_page_size)[None, :], + (vals // physical_page_size)[None, :], mask=real_mask, ) @@ -210,7 +210,7 @@ def fused_dsa_draft_extend_metadata( total_len: int, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, max_extend_len: int, max_total_len: int, static_extend_len: bool = False, @@ -245,7 +245,7 @@ def fused_dsa_draft_extend_metadata( assert total_len <= bs * max_extend_len assert index_kpool > 0 - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -297,7 +297,7 @@ def fused_dsa_draft_extend_metadata( num_splits, dsa_index_topk, index_kpool, - real_page_size, + physical_page_size, has_real_page_table, has_page_table_1, static_extend_len, diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py index 0d49743ed8..82f7a8e673 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py +++ b/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py @@ -44,7 +44,7 @@ def _fused_dsa_target_verify_metadata_kernel( num_splits, dsa_index_topk: tl.constexpr, index_kpool: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, next_n: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGED_MQA_CTX_LENS: tl.constexpr, @@ -146,13 +146,13 @@ def _fused_dsa_target_verify_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + out_row_i64 * real_page_table_stride_0 + real_cols * real_page_table_stride_1, - vals // real_page_size, + vals // physical_page_size, mask=real_mask, ) @@ -171,7 +171,7 @@ def _prep_fused_dsa_target_verify_metadata_launch( bs: int, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, next_n: int, paged_mqa_ctx_lens_2d: torch.Tensor = None, index_kpool: int = 1, @@ -189,7 +189,7 @@ def _prep_fused_dsa_target_verify_metadata_launch( assert next_n > 0 assert index_kpool > 0 - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -251,7 +251,7 @@ def _prep_fused_dsa_target_verify_metadata_launch( num_splits, dsa_index_topk, index_kpool, - real_page_size, + physical_page_size, next_n, has_real_page_table, has_paged_mqa_ctx_lens, @@ -279,7 +279,7 @@ def fused_dsa_target_verify_metadata( bs: int, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, next_n: int, paged_mqa_ctx_lens_2d: torch.Tensor = None, index_kpool: int = 1, @@ -305,7 +305,7 @@ def fused_dsa_target_verify_metadata( bs, max_seqlen_k, dsa_index_topk, - real_page_size, + physical_page_size, next_n, paged_mqa_ctx_lens_2d, index_kpool, diff --git a/python/sglang/kernels/ops/attention/dsa_metadata.py b/python/sglang/kernels/ops/attention/dsa_metadata.py index df430081a1..858dc1825a 100644 --- a/python/sglang/kernels/ops/attention/dsa_metadata.py +++ b/python/sglang/kernels/ops/attention/dsa_metadata.py @@ -33,7 +33,7 @@ def _fused_dsa_decode_metadata_kernel( bs: tl.constexpr, max_len, dsa_index_topk: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGE_TABLE_1: tl.constexpr, BLOCK_BS: tl.constexpr, @@ -98,13 +98,13 @@ def _fused_dsa_decode_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + row.to(tl.int64) * real_page_table_stride_0 + real_cols * real_page_table_stride_1, - vals // real_page_size, + vals // physical_page_size, mask=real_mask, ) @@ -122,15 +122,15 @@ def fused_dsa_decode_metadata( bs: int, max_len: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, ) -> None: """Fill decode-graph DSA metadata (seqlens + page tables) from req_to_token. ``page_table_1`` (the wide page_size=1 table) is optional: pass ``None`` to skip materializing it and write only the compact ``real_page_table`` - (page_size=``real_page_size``). This is used by the fused decode CUDA graph, + (page_size=``physical_page_size``). This is used by the fused decode CUDA graph, where the wide table is never read (attention uses topk_indices, the indexer - uses real_page_table); ``real_page_size`` must be >1 in that case. When a + uses real_page_table); ``physical_page_size`` must be >1 in that case. When a tensor is passed, behavior is unchanged (both tables are written). Contract: each page-table row is written only over its live prefix @@ -150,7 +150,7 @@ def fused_dsa_decode_metadata( dsa_cu_seqlens_k[:1].zero_() return - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -194,7 +194,7 @@ def fused_dsa_decode_metadata( bs, max_len, dsa_index_topk, - real_page_size, + physical_page_size, has_real_page_table, has_page_table_1, BLOCK_BS=block_bs, @@ -234,7 +234,7 @@ def _fused_dsa_target_verify_metadata_kernel( bs: tl.constexpr, max_seqlen_k, dsa_index_topk: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, next_n: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGED_MQA_CTX_LENS: tl.constexpr, @@ -330,13 +330,13 @@ def _fused_dsa_target_verify_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + out_row.to(tl.int64) * real_page_table_stride_0 + real_cols * real_page_table_stride_1, - vals // real_page_size, + vals // physical_page_size, mask=real_mask, ) @@ -355,7 +355,7 @@ def fused_dsa_target_verify_metadata( bs: int, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, next_n: int, paged_mqa_ctx_lens_2d: torch.Tensor = None, ) -> None: @@ -374,7 +374,7 @@ def fused_dsa_target_verify_metadata( return assert next_n > 0 - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -433,7 +433,7 @@ def fused_dsa_target_verify_metadata( bs, max_seqlen_k, dsa_index_topk, - real_page_size, + physical_page_size, next_n, has_real_page_table, has_paged_mqa_ctx_lens, @@ -477,7 +477,7 @@ def _fused_dsa_draft_extend_metadata_kernel( total_len, max_seqlen_k, dsa_index_topk: tl.constexpr, - real_page_size: tl.constexpr, + physical_page_size: tl.constexpr, HAS_REAL_PAGE_TABLE: tl.constexpr, HAS_PAGE_TABLE_1: tl.constexpr, STATIC_EXTEND_LEN: tl.constexpr, @@ -603,13 +603,13 @@ def _fused_dsa_draft_extend_metadata_kernel( ) if HAS_REAL_PAGE_TABLE: - real_mask = mask & ((offs_n[None, :] % real_page_size) == 0) - real_cols = offs_n // real_page_size + real_mask = mask & ((offs_n[None, :] % physical_page_size) == 0) + real_cols = offs_n // physical_page_size tl.store( real_page_table + out_rows.to(tl.int64)[:, None] * real_page_table_stride_0 + real_cols[None, :] * real_page_table_stride_1, - (vals // real_page_size)[None, :], + (vals // physical_page_size)[None, :], mask=real_mask, ) @@ -630,7 +630,7 @@ def fused_dsa_draft_extend_metadata( total_len: int, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, max_extend_len: int, max_total_len: int, static_extend_len: bool = False, @@ -663,7 +663,7 @@ def fused_dsa_draft_extend_metadata( assert max_extend_len > 0 assert total_len <= bs * max_extend_len - has_real_page_table = real_page_size > 1 + has_real_page_table = physical_page_size > 1 if has_real_page_table: assert real_page_table is not None assert real_page_table.is_cuda @@ -712,7 +712,7 @@ def fused_dsa_draft_extend_metadata( total_len, max_seqlen_k, dsa_index_topk, - real_page_size, + physical_page_size, has_real_page_table, has_page_table_1, static_extend_len, diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index bfa2fd668e..881ff45493 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -141,12 +141,14 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: ) else: index_kpool = get_dsa_index_kpool(hf_config) - if index_kpool > 1 and getattr(cfg, "dcp_size", 1) > 1: + if index_kpool > 1 and cfg.dcp_size > 1: raise ValueError( "--dcp-size > 1 is not supported for DSA with " f"index_kpool={index_kpool}: a pooled index key spans " "positions owned by different DCP ranks." ) + # A logical page must own a whole pooled index group so that a + # prefix hit never ends inside one. page_size = 64 * index_kpool overrides["page_size"] = page_size logger.warning(f"Setting page size to {page_size} for DeepSeek DSA.") diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py index 5cc0f907db..15fe941ec6 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py @@ -67,8 +67,8 @@ class DeepseekSparseAttnBackendKPoolMixin: return "fa3" return dsa_impl - def _kpool_slots_per_page(self) -> int: - return self.token_to_kv_pool.slots_per_page + def _kpool_index_page_size(self) -> int: + return self.token_to_kv_pool.index_page_size def _build_kpool_paged_mqa_schedule_metadata(self) -> bool: if self.device_sm_major == 9: @@ -86,7 +86,7 @@ class DeepseekSparseAttnBackendKPoolMixin: return metadata forward_mode = forward_batch.forward_mode - slots_per_page = self._kpool_slots_per_page() + index_page_size = self._kpool_index_page_size() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() if forward_mode.is_extend_without_speculative(): assert topk_transform_method is not None @@ -94,9 +94,9 @@ class DeepseekSparseAttnBackendKPoolMixin: return init_kpool_extend_metadata( metadata, forward_batch, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, + index_page_size=index_page_size, topk_transform_method=topk_transform_method, full_real_page_table=kpool_inputs.full_real_page_table, full_seqlens_expanded=kpool_inputs.full_seqlens_expanded, @@ -106,9 +106,9 @@ class DeepseekSparseAttnBackendKPoolMixin: metadata, metadata.cache_seqlens_int32, forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -116,12 +116,12 @@ class DeepseekSparseAttnBackendKPoolMixin: return init_kpool_write_plan( metadata, forward_batch, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, real_page_table=metadata.real_page_table, num_draft_tokens=1, write_start=(forward_batch.seq_lens - 1).to(torch.int32), - slots_per_page=slots_per_page, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -129,12 +129,12 @@ class DeepseekSparseAttnBackendKPoolMixin: return init_kpool_write_plan( metadata, forward_batch, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, real_page_table=metadata.real_page_table, num_draft_tokens=self.speculative_num_draft_tokens, write_start=forward_batch.seq_lens.to(torch.int32), - slots_per_page=slots_per_page, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -149,14 +149,14 @@ class DeepseekSparseAttnBackendKPoolMixin: return init_kpool_write_plan( metadata, forward_batch, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, real_page_table=metadata.real_page_table, num_draft_tokens=self.speculative_num_draft_tokens, write_start=( forward_batch.seq_lens - self.speculative_num_draft_tokens ).to(torch.int32), - slots_per_page=slots_per_page, + index_page_size=index_page_size, effective_n_per_batch=effective_n_per_batch, build_schedule_metadata=build_schedule_metadata, ) @@ -169,15 +169,15 @@ class DeepseekSparseAttnBackendKPoolMixin: if self.dsa_index_kpool <= 1: return metadata - slots_per_page = self._kpool_slots_per_page() + index_page_size = self._kpool_index_page_size() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() metadata = init_pooled_paged_mqa_metadata( metadata, metadata.cache_seqlens_int32, forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -191,14 +191,14 @@ class DeepseekSparseAttnBackendKPoolMixin: metadata = init_kpool_write_plan_capture( metadata, max_bs=bs, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, num_draft_tokens=( 1 if is_decode else self.speculative_num_draft_tokens ), device=self.device, is_verify=not is_decode, - slots_per_page=slots_per_page, + index_page_size=index_page_size, is_v2=is_v2, build_schedule_metadata=build_schedule_metadata, ) @@ -216,15 +216,15 @@ class DeepseekSparseAttnBackendKPoolMixin: if self.dsa_index_kpool <= 1: return - slots_per_page = self._kpool_slots_per_page() + index_page_size = self._kpool_index_page_size() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() update_pooled_paged_mqa_metadata( metadata, metadata.cache_seqlens_int32, forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -250,11 +250,11 @@ class DeepseekSparseAttnBackendKPoolMixin: write_start=write_start, req_pool_indices=req_pool_indices, real_page_table=metadata.real_page_table, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, num_draft_tokens=(1 if is_decode else self.speculative_num_draft_tokens), forward_mode=forward_mode, - slots_per_page=slots_per_page, + index_page_size=index_page_size, effective_n_per_batch=effective_n_per_batch, ) @@ -267,15 +267,15 @@ class DeepseekSparseAttnBackendKPoolMixin: if self.dsa_index_kpool <= 1: return - slots_per_page = self._kpool_slots_per_page() + index_page_size = self._kpool_index_page_size() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() update_pooled_paged_mqa_metadata( metadata, precomputed.cache_seqlens, forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, + index_page_size=index_page_size, build_schedule_metadata=build_schedule_metadata, ) @@ -294,9 +294,9 @@ class DeepseekSparseAttnBackendKPoolMixin: write_start=write_start, req_pool_indices=precomputed.req_pool_indices, real_page_table=metadata.real_page_table, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, + kpool=self.dsa_index_kpool, + physical_page_size=self.physical_page_size, num_draft_tokens=self.speculative_num_draft_tokens if is_verify else 1, forward_mode=forward_mode, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py index 9daea339c9..e967b0854e 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py @@ -134,8 +134,10 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: dsa_cu_seqlens_k = torch.empty( bs + 1, dtype=torch.int32, device=self.device ) - if self.real_page_size > 1: - real_cols = (max_len + self.real_page_size - 1) // self.real_page_size + if self.physical_page_size > 1: + real_cols = ( + max_len + self.physical_page_size - 1 + ) // self.physical_page_size real_page_table = torch.empty( (bs, real_cols), dtype=torch.int32, device=self.device ) @@ -157,7 +159,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: bs=bs, max_len=max_len, dsa_index_topk=self.dsa_index_topk, - real_page_size=self.real_page_size, + physical_page_size=self.physical_page_size, ) seqlens_expanded = cache_seqlens seqlens_expanded_size = bs @@ -204,7 +206,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: dsa_cu_seqlens_k = compute_cu_seqlens(dsa_cache_seqlens) # Transform page table if needed - if self.real_page_size > 1: + if self.physical_page_size > 1: real_page_table = self._transform_table_1_to_real(page_indices) else: real_page_table = None # Will use page_indices directly @@ -264,10 +266,10 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: dtype=torch.int32, device=self.device, ) - if self.real_page_size > 1: + if self.physical_page_size > 1: real_cols = ( - max_seqlen_k + self.real_page_size - 1 - ) // self.real_page_size + max_seqlen_k + self.physical_page_size - 1 + ) // self.physical_page_size real_page_table = torch.empty( (seqlens_expanded_size, real_cols), dtype=torch.int32, @@ -292,7 +294,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: bs=bs, max_seqlen_k=max_seqlen_k, dsa_index_topk=self.dsa_index_topk, - real_page_size=self.real_page_size, + physical_page_size=self.physical_page_size, next_n=self.speculative_num_draft_tokens, ) @@ -355,7 +357,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin: dsa_cu_seqlens_k = compute_cu_seqlens(dsa_cache_seqlens) # Transform page table - if self.real_page_size > 1: + if self.physical_page_size > 1: real_page_table = self._transform_table_1_to_real(page_indices) else: real_page_table = None diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index a2259755ad..33a8adb86f 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -421,7 +421,7 @@ class IndexerKPool(MultiPlatformOp): max_req = int(active_req.max().item()) min_loc = int(active_write_loc.min().item()) max_loc = int(active_write_loc.max().item()) - max_page = max_loc // int(pool.slots_per_page) + max_page = max_loc // int(pool.index_page_size) if ( min_chunk < 0 or max_chunk >= key.shape[0] @@ -439,7 +439,7 @@ class IndexerKPool(MultiPlatformOp): f"{key.shape=}, {gate_score.shape=}, {tail_k_buf.shape=}, " f"{pool.get_index_k_with_scale_buffer(layer_id=layer_id).shape=}, " f"{min_chunk=}, {max_chunk=}, {min_req=}, {max_req=}, " - f"{min_loc=}, {max_loc=}, {pool.slots_per_page=}" + f"{min_loc=}, {max_loc=}, {pool.index_page_size=}" ) if not tails.is_empty: tail_chunk_max = tails.chunk_src.to(torch.long) + torch.clamp( @@ -778,7 +778,7 @@ class IndexerKPool(MultiPlatformOp): return topk_from_pooled_history_logits( logits=logits, group_lengths=pool_lens, - pool_size=self.index_kpool, + kpool=self.index_kpool, topk=self.index_topk, page_table=page_table, topk_offsets=topk_offsets, @@ -960,7 +960,7 @@ class IndexerKPool(MultiPlatformOp): assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) pool = get_token_to_kv_pool() - page_size = pool.slots_per_page + page_size = pool.index_page_size # DeepGEMM paged-MQA requires 64-token pages. assert page_size == 64, "only support page size 64" @@ -1245,11 +1245,11 @@ class IndexerKPool(MultiPlatformOp): and forward_batch.extend_seq_lens_cpu is not None ) - pool_size = self.index_kpool + kpool = self.index_kpool page_size = metadata.attn_metadata.page_size - slots_per_page = get_token_to_kv_pool().slots_per_page + index_page_size = get_token_to_kv_pool().index_page_size token_nums = q_fp8.shape[0] - tail_pool = pool_size - 1 + tail_pool = kpool - 1 topk_result = torch.empty( (token_nums, self.index_topk + tail_pool), device=q_fp8.device, @@ -1278,7 +1278,7 @@ class IndexerKPool(MultiPlatformOp): continue seq_len = int(forward_batch.seq_lens_cpu[i].item()) - pool_seq_len = seq_len // pool_size + pool_seq_len = seq_len // kpool req_pool_idx = int(forward_batch.req_pool_indices[i].item()) if ( cache_write_stream is not None @@ -1293,7 +1293,7 @@ class IndexerKPool(MultiPlatformOp): local_seqlens = seq_lens_expanded[q_slice] if pooled_seq_lens_expanded is None: local_pool_lens = torch.div( - local_seqlens, pool_size, rounding_mode="floor" + local_seqlens, kpool, rounding_mode="floor" ).to(torch.int32) else: local_pool_lens = pooled_seq_lens_expanded[q_slice] @@ -1362,10 +1362,10 @@ class IndexerKPool(MultiPlatformOp): i, :num_token_pages ].contiguous() pool_pages = ( - curr_pool_start + slots_per_page - 1 - ) // slots_per_page + curr_pool_start + index_page_size - 1 + ) // index_page_size pooled_page_table = build_pooled_page_table_64( - token_page_table, pool_size + token_page_table, kpool )[:pool_pages].contiguous() k_u8 = torch.empty( (pool_seq_len, self.head_dim), @@ -1404,10 +1404,10 @@ class IndexerKPool(MultiPlatformOp): i, :num_token_pages ].contiguous() pool_pages = ( - pool_seq_len + slots_per_page - 1 - ) // slots_per_page + pool_seq_len + index_page_size - 1 + ) // index_page_size pooled_page_table = build_pooled_page_table_64( - token_page_table, pool_size + token_page_table, kpool )[:pool_pages].contiguous() seq_len_t = torch.tensor( [pool_seq_len], dtype=torch.int32, device=q_fp8.device @@ -1504,7 +1504,7 @@ class IndexerKPool(MultiPlatformOp): assert forward_batch.forward_mode.is_extend_without_speculative() - page_size = get_token_to_kv_pool().slots_per_page + page_size = get_token_to_kv_pool().index_page_size assert page_size == 64, "only support page size 64" assert len(weights.shape) == 3 weights = weights.squeeze(-1) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py index bc783b6c60..733df7a87f 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py @@ -29,12 +29,9 @@ _is_hip = is_hip() logger = logging.getLogger(__name__) -def kpool_metadata_fusion_supported(pool_size, page_size, topk): +def kpool_metadata_fusion_supported(kpool, page_size, topk): return ( - pool_size > 1 - and page_size == 64 - and page_size % pool_size == 0 - and topk % pool_size == 0 + kpool > 1 and page_size == 64 and page_size % kpool == 0 and topk % kpool == 0 ) @@ -44,7 +41,7 @@ class DSAMetadataManagementMixin: def _init_kpool_metadata_fusion(self): requested = envs.SGLANG_EXPERIMENTAL_DSA_KPOOL_METADATA_FUSION.get() supported = kpool_metadata_fusion_supported( - self.dsa_index_kpool, self.real_page_size, self.dsa_index_topk + self.dsa_index_kpool, self.physical_page_size, self.dsa_index_topk ) self.experimental_kpool_metadata_fusion = ( requested and supported and is_cuda() and not is_hip() diff --git a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py index 3edf5c2556..614b93ef22 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py @@ -48,23 +48,21 @@ def _kpool_cache_k_offsets( return page * BUF_NUMEL_PER_PAGE + slot * HEAD_DIM + cols -def kpool_max_closed_pools(num_draft_tokens: int, pool_size: int) -> int: - return (num_draft_tokens + pool_size - 1) // pool_size +def kpool_max_closed_pools(num_draft_tokens: int, kpool: int) -> int: + return (num_draft_tokens + kpool - 1) // kpool def build_pooled_page_table_64( page_table_64: torch.Tensor, - pool_size: int, + kpool: int, ) -> torch.Tensor: # Advanced indexing is required: a (1, 1) strided slice can remain non-unit- # stride even after contiguous(), which DeepGEMM rejects. - assert BLOCK_SIZE_K % pool_size == 0, ( - f"pool_size ({pool_size}) must divide page_size ({BLOCK_SIZE_K})" + assert BLOCK_SIZE_K % kpool == 0, ( + f"kpool ({kpool}) must divide page_size ({BLOCK_SIZE_K})" ) - idx = torch.arange( - 0, page_table_64.shape[-1], pool_size, device=page_table_64.device - ) - return page_table_64[..., idx] // pool_size + idx = torch.arange(0, page_table_64.shape[-1], kpool, device=page_table_64.device) + return page_table_64[..., idx] // kpool def gather_index_k_scale_prefix_into( @@ -79,7 +77,7 @@ def gather_index_k_scale_prefix_into( assert page_indices.dtype in (torch.int32, torch.int64) assert k_out.dtype == torch.uint8 assert scale_out.dtype == torch.float32 - assert pool.slots_per_page == BLOCK_SIZE_K + assert pool.index_page_size == BLOCK_SIZE_K assert k_out.shape[0] >= seq_len assert k_out.shape[1] == INDEX_HEAD_DIM assert scale_out.shape[0] >= seq_len @@ -96,11 +94,11 @@ def gather_index_k_scale_prefix_into( page_indices, k_out, scale_out, - PAGE_SIZE=pool.slots_per_page, + PAGE_SIZE=pool.index_page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), HEAD_DIM=INDEX_HEAD_DIM, - S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * INDEX_HEAD_DIM, + S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * INDEX_HEAD_DIM, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), ) @@ -154,10 +152,10 @@ def kpool_build_ragged_layout( cu_q_len_excl: torch.Tensor, ragged_q_len: torch.Tensor, pooled_seq_lens_expanded: torch.Tensor, - slots_per_page: int, + index_page_size: int, total_pool_pages: int, total_q: int, - pool_size: int, + kpool: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: device = full_page_table.device n_rag = cu_pages_excl.shape[0] @@ -181,8 +179,8 @@ def kpool_build_ragged_layout( q_ks, q_ke, max_pool_pages, - slots_per_page, - pool_size, + index_page_size, + kpool, BLOCK_PAGE=128, BLOCK_Q=128, ) @@ -201,7 +199,7 @@ def _kpool_build_ragged_layout_kernel( q_ks_ptr, q_ke_ptr, MAX_POOL_PAGES, - SLOTS_PER_PAGE: tl.constexpr, + INDEX_PAGE_SIZE: tl.constexpr, POOL_SIZE: tl.constexpr, BLOCK_PAGE: tl.constexpr, BLOCK_Q: tl.constexpr, @@ -211,7 +209,7 @@ def _kpool_build_ragged_layout_kernel( n_pages = tl.load(ragged_pool_pages_ptr + k) q_start = tl.load(cu_q_len_excl_ptr + k) q_count = tl.load(ragged_q_len_ptr + k) - ks_val = page_start * SLOTS_PER_PAGE + ks_val = page_start * INDEX_PAGE_SIZE for p_off in tl.range(0, BLOCK_PAGE * tl.cdiv(n_pages, BLOCK_PAGE), BLOCK_PAGE): p_offs = p_off + tl.arange(0, BLOCK_PAGE) @@ -234,7 +232,7 @@ def _kpool_build_ragged_layout_kernel( plen = tl.load(pooled_seq_lens_ptr + q_start + q_offs, mask=q_mask, other=0) ke_val = tl.minimum( ks_val + plen, - (page_start + n_pages) * SLOTS_PER_PAGE, + (page_start + n_pages) * INDEX_PAGE_SIZE, ) tl.store( q_ks_ptr + q_start + q_offs, @@ -255,13 +253,13 @@ def _prep_update_kpool_write_plan_launch( pool_seqlens_per_q_out: Optional[torch.Tensor], seqlens_per_q_out: Optional[torch.Tensor], *, - pool_size: int, + kpool: int, num_draft_tokens: int, - slots_per_page: int, + index_page_size: int, ): # Eager and captured replay must share one launch spec, including - # N > pool_size where one write closes multiple pools. - max_closed_pools = kpool_max_closed_pools(num_draft_tokens, pool_size) + # N > kpool where one write closes multiple pools. + max_closed_pools = kpool_max_closed_pools(num_draft_tokens, kpool) bs = write_start.shape[0] assert write_loc_out.shape == (bs, max_closed_pools), write_loc_out.shape @@ -292,9 +290,9 @@ def _prep_update_kpool_write_plan_launch( write_loc_out.stride(0), ) constexprs = dict( - POOL_SIZE=pool_size, + POOL_SIZE=kpool, N=num_draft_tokens, - SLOTS_PER_PAGE=slots_per_page, + INDEX_PAGE_SIZE=index_page_size, MAX_CLOSED_POOLS=max_closed_pools, HAS_PER_Q=has_per_q_outputs, ) @@ -312,9 +310,9 @@ def update_kpool_write_plan_cuda_graph( pool_seqlens_per_q_out: Optional[torch.Tensor], seqlens_per_q_out: Optional[torch.Tensor], *, - pool_size: int, + kpool: int, num_draft_tokens: int, - slots_per_page: int, + index_page_size: int, ) -> None: if write_start.shape[0] == 0: return @@ -328,9 +326,9 @@ def update_kpool_write_plan_cuda_graph( write_loc_out, pool_seqlens_per_q_out, seqlens_per_q_out, - pool_size=pool_size, + kpool=kpool, num_draft_tokens=num_draft_tokens, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) _update_kpool_write_plan_kernel[(bs,)](*args, **constexprs) @@ -351,7 +349,7 @@ def _update_kpool_write_plan_kernel( write_loc_out_stride_0, POOL_SIZE: tl.constexpr, N: tl.constexpr, - SLOTS_PER_PAGE: tl.constexpr, + INDEX_PAGE_SIZE: tl.constexpr, MAX_CLOSED_POOLS: tl.constexpr, HAS_PER_Q: tl.constexpr, ): @@ -372,7 +370,7 @@ def _update_kpool_write_plan_kernel( tl.store(tail_logical_start_out_ptr + b, (base_pool * POOL_SIZE).to(tl.int32)) for p in tl.static_range(0, MAX_CLOSED_POOLS): pool_id = base_pool + p - pool_page_group = pool_id // SLOTS_PER_PAGE + pool_page_group = pool_id // INDEX_PAGE_SIZE token_page_row = pool_page_group * POOL_SIZE token_page_row = tl.minimum( tl.maximum(token_page_row, 0), real_page_table_cols - 1 @@ -387,7 +385,7 @@ def _update_kpool_write_plan_kernel( ).to(tl.int64) // POOL_SIZE ) - write_loc = packed_page * SLOTS_PER_PAGE + (pool_id % SLOTS_PER_PAGE) + write_loc = packed_page * INDEX_PAGE_SIZE + (pool_id % INDEX_PAGE_SIZE) tl.store( write_loc_out_ptr + b * write_loc_out_stride_0 + p, write_loc.to(tl.int64), @@ -397,45 +395,45 @@ def _update_kpool_write_plan_kernel( def compute_pooled_write_locs( page_table_64: torch.Tensor, pool_ids: torch.Tensor, - pool_size: int, + kpool: int, ) -> torch.Tensor: assert page_table_64.ndim == 1 pool_ids = pool_ids.to(torch.int64) pool_page_group = torch.div(pool_ids, BLOCK_SIZE_K, rounding_mode="floor") - token_page_row = pool_page_group * pool_size + token_page_row = pool_page_group * kpool packed_page = ( page_table_64.index_select(0, token_page_row.to(torch.int64)).to(torch.int64) - // pool_size + // kpool ) return packed_page * BLOCK_SIZE_K + torch.remainder(pool_ids, BLOCK_SIZE_K) -def history_group_budget_for_topk(topk: int, pool_size: int) -> int: - assert topk % pool_size == 0 - return topk // pool_size +def history_group_budget_for_topk(topk: int, kpool: int) -> int: + assert topk % kpool == 0 + return topk // kpool def expand_pooled_groups_to_topk( group_ids: torch.Tensor, group_valid: torch.Tensor, topk: int, - pool_size: int, + kpool: int, page_table: torch.Tensor | None = None, topk_offsets: torch.Tensor | None = None, ) -> torch.Tensor: assert group_ids.ndim == 2 assert group_valid.shape == group_ids.shape - assert topk % pool_size == 0 - assert group_ids.shape[1] == history_group_budget_for_topk(topk, pool_size) + assert topk % kpool == 0 + assert group_ids.shape[1] == history_group_budget_for_topk(topk, kpool) assert page_table is None or topk_offsets is None device = group_ids.device - offsets = torch.arange(pool_size, device=device, dtype=torch.int64) - token_ids = group_ids.to(torch.int64).unsqueeze(-1) * pool_size + offsets + offsets = torch.arange(kpool, device=device, dtype=torch.int64) + token_ids = group_ids.to(torch.int64).unsqueeze(-1) * kpool + offsets token_ids = token_ids.reshape(group_ids.shape[0], topk) valid = ( group_valid.unsqueeze(-1) - .expand(-1, -1, pool_size) + .expand(-1, -1, kpool) .reshape(group_ids.shape[0], topk) ) @@ -460,7 +458,7 @@ def append_kpool_tail_to_topk( topk_result: torch.Tensor, seq_lens: torch.Tensor, pool_lens: torch.Tensor, - pool_size: int, + kpool: int, page_table: torch.Tensor | None = None, topk_offsets: torch.Tensor | None = None, ) -> torch.Tensor: @@ -470,7 +468,7 @@ def append_kpool_tail_to_topk( assert seq_lens.shape[0] == topk_result.shape[0] assert pool_lens.shape[0] == topk_result.shape[0] - tail_pool = pool_size - 1 + tail_pool = kpool - 1 if tail_pool == 0: return topk_result @@ -516,7 +514,7 @@ def append_kpool_tail_to_topk( N_COLS=n_cols, OUT_COLS=out_cols, PAGE_TABLE_COLS=page_table_cols, - POOL_SIZE=pool_size, + POOL_SIZE=kpool, HAS_PAGE_TABLE=has_page_table, HAS_TOPK_OFFSETS=has_topk_offsets, BLOCK_COLS=block_cols, @@ -591,7 +589,7 @@ def _append_kpool_tail_to_topk_kernel( def topk_from_pooled_history_logits( logits: torch.Tensor, group_lengths: torch.Tensor, - pool_size: int, + kpool: int, topk: int, page_table: torch.Tensor | None = None, topk_offsets: torch.Tensor | None = None, @@ -604,12 +602,12 @@ def topk_from_pooled_history_logits( assert group_lengths.ndim == 1 assert logits.shape[0] == group_lengths.shape[0] assert topk > 0 - assert topk % pool_size == 0 + assert topk % kpool == 0 assert out_rows is None or out_rows >= logits.shape[0] assert page_table_row_index is None or page_table is not None _, cols = logits.shape - group_topk = history_group_budget_for_topk(topk, pool_size) + group_topk = history_group_budget_for_topk(topk, kpool) if topk_offsets is not None and topk_offsets.ndim == 2: assert topk_offsets.shape[1] == 1 topk_offsets = topk_offsets.squeeze(1) @@ -618,7 +616,7 @@ def topk_from_pooled_history_logits( raise NotImplementedError( "index_kpool topk only supports pooled group_topk in " f"(128, 160, 192, 224, 256, 512, 2048), got {group_topk} " - f"(topk={topk}, pool_size={pool_size})." + f"(topk={topk}, kpool={kpool})." ) if not logits.is_cuda or logits.dtype != torch.float32: raise NotImplementedError( @@ -634,7 +632,7 @@ def topk_from_pooled_history_logits( result = fast_kpool_topk_transform_fused( score=logits, lengths=group_lengths.to(torch.int32), - pool_size=pool_size, + kpool=kpool, topk=topk, page_table=page_table, topk_indices_offset=topk_offsets, @@ -674,7 +672,7 @@ def topk_from_pooled_history_logits( selected_groups.contiguous(), group_valid, topk=topk, - pool_size=pool_size, + kpool=kpool, page_table=page_table, topk_offsets=topk_offsets, ) @@ -685,7 +683,7 @@ def topk_from_pooled_history_logits( expanded, seq_lens=seq_lens, pool_lens=group_lengths, - pool_size=pool_size, + kpool=kpool, page_table=page_table, topk_offsets=topk_offsets, ) @@ -718,7 +716,7 @@ def kpool_softmax_rotate_write_cache( assert slot_score.dtype in KPOOL_SCORE_DTYPES assert ape.dtype == torch.float32 assert buf.dtype == torch.uint8 - assert pool.slots_per_page == BLOCK_SIZE_K + assert pool.index_page_size == BLOCK_SIZE_K assert pool.index_head_dim == INDEX_HEAD_DIM assert loc.dtype == torch.int64 assert write_cache or return_compressed @@ -777,12 +775,12 @@ def kpool_softmax_rotate_write_cache( slot_score.stride(0), slot_score.stride(1), ape.stride(0), - PAGE_SIZE=pool.slots_per_page, + PAGE_SIZE=pool.index_page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), POOL_SIZE=slot_k.shape[1], HEAD_DIM=slot_k.shape[2], - S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim, + S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * pool.index_head_dim, ROUND_SCALE=round_scale, HAS_WRITE_MASK=has_write_mask, RETURN_COMPRESSED=return_compressed, @@ -822,7 +820,7 @@ def kpool_decode_update_and_maybe_write_cache( assert slot_score.dtype == tail_score.dtype assert ape.dtype == torch.float32 assert buf.dtype == torch.uint8 - assert pool.slots_per_page == BLOCK_SIZE_K + assert pool.index_page_size == BLOCK_SIZE_K assert pool.index_head_dim == INDEX_HEAD_DIM assert tail_k.is_contiguous() assert tail_score.is_contiguous() @@ -878,10 +876,10 @@ def kpool_decode_update_and_maybe_write_cache( TAIL_SIZE=tail_k.shape[1], HEAD_DIM=tail_k.shape[2], BLOCK_TABLE_COLS=block_tables.shape[1], - S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim, + S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * pool.index_head_dim, ROUND_SCALE=round_scale, BLOCK_D=triton.next_power_of_2(tail_k.shape[2]), - SLOTS_PER_PAGE=pool.slots_per_page, + INDEX_PAGE_SIZE=pool.index_page_size, ) @@ -1063,7 +1061,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( PRESHUFFLE_TILE: tl.constexpr, ROUND_SCALE: tl.constexpr, BLOCK_D: tl.constexpr, - SLOTS_PER_PAGE: tl.constexpr, + INDEX_PAGE_SIZE: tl.constexpr, ): row = tl.program_id(0) offs = tl.arange(0, BLOCK_D) @@ -1163,7 +1161,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( quantized = tl.minimum(tl.maximum(quantized, fp8_min), fp8_max) pool_id = safe_pos // POOL_SIZE - pool_page_group = pool_id // SLOTS_PER_PAGE + pool_page_group = pool_id // INDEX_PAGE_SIZE token_page_row = pool_page_group * POOL_SIZE token_page_row = tl.minimum(tl.maximum(token_page_row, 0), BLOCK_TABLE_COLS - 1) packed_page = tl.load( @@ -1172,7 +1170,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel( + token_page_row * block_tables_stride_1, ) loc_page_index = packed_page.to(tl.int64) // POOL_SIZE - loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE + loc_token_offset_in_page = pool_id % INDEX_PAGE_SIZE out_k_offsets = _kpool_cache_k_offsets( loc_page_index, loc_token_offset_in_page, @@ -1243,7 +1241,7 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel( ROUND_SCALE: tl.constexpr, HAS_WRITE_MASK: tl.constexpr, BLOCK_D: tl.constexpr, - SLOTS_PER_PAGE: tl.constexpr, + INDEX_PAGE_SIZE: tl.constexpr, ): row = tl.program_id(0) if HAS_WRITE_MASK: @@ -1285,8 +1283,8 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel( quantized, scale = _hadamard_quantize_fp8(acc, denom, ROUND_SCALE) loc = tl.load(loc_ptr + row) - loc_page_index = loc // SLOTS_PER_PAGE - loc_token_offset_in_page = loc % SLOTS_PER_PAGE + loc_page_index = loc // INDEX_PAGE_SIZE + loc_token_offset_in_page = loc % INDEX_PAGE_SIZE out_k_offsets = _kpool_cache_k_offsets( loc_page_index, loc_token_offset_in_page, @@ -1321,7 +1319,7 @@ def kpool_assemble_softmax_rotate_write_cache( write_mask: torch.Tensor | None = None, round_scale: bool = False, ) -> None: - pool_size = pool.index_kpool + kpool = pool.index_kpool n_pools = req_pool_idx.shape[0] if n_pools == 0: return @@ -1339,7 +1337,7 @@ def kpool_assemble_softmax_rotate_write_cache( buf_fp8 = buf.view(torch.float8_e4m3fn) buf_fp32 = buf.view(torch.float32) - slots_per_page = pool.slots_per_page + index_page_size = pool.index_page_size _kpool_assemble_softmax_rotate_write_cache_kernel[(n_pools,)]( buf_fp8, @@ -1361,14 +1359,14 @@ def kpool_assemble_softmax_rotate_write_cache( ape.stride(0), BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), - POOL_SIZE=pool_size, + POOL_SIZE=kpool, TAIL_SIZE=tail_k.shape[1], HEAD_DIM=INDEX_HEAD_DIM, - S_OFFSET_NBYTES_IN_PAGE=slots_per_page * INDEX_HEAD_DIM, + S_OFFSET_NBYTES_IN_PAGE=index_page_size * INDEX_HEAD_DIM, ROUND_SCALE=round_scale, HAS_WRITE_MASK=has_write_mask, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), - SLOTS_PER_PAGE=slots_per_page, + INDEX_PAGE_SIZE=index_page_size, ) @@ -1383,14 +1381,14 @@ def scatter_kpool_tail_updates( chunk_src_start: torch.Tensor, n_write: torch.Tensor, ) -> None: - pool_size = pool.index_kpool + kpool = pool.index_kpool n_rows = req_pool_idx.shape[0] if n_rows == 0: return chunk_k = chunk_k.contiguous() chunk_score = chunk_score.contiguous() - _scatter_kpool_tail_updates_kernel[(n_rows, pool_size)]( + _scatter_kpool_tail_updates_kernel[(n_rows, kpool)]( chunk_k, chunk_score, tail_k, @@ -1402,7 +1400,7 @@ def scatter_kpool_tail_updates( chunk_k.stride(0), tail_k.stride(0), tail_k.stride(1), - POOL_SIZE=pool_size, + POOL_SIZE=kpool, TAIL_SIZE=tail_k.shape[1], HEAD_DIM=INDEX_HEAD_DIM, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), @@ -1458,7 +1456,7 @@ def _pack_pool_slots_to_payload_kernel( locs_ptr, payload_ptr, payload_bytes: tl.constexpr, - slots_per_page: tl.constexpr, + index_page_size: tl.constexpr, head_dim: tl.constexpr, page_bytes: tl.constexpr, scale_region_off: tl.constexpr, @@ -1466,8 +1464,8 @@ def _pack_pool_slots_to_payload_kernel( ): row = tl.program_id(0) loc = tl.load(locs_ptr + row).to(tl.int64) - page = loc // slots_per_page - slot = loc % slots_per_page + page = loc // index_page_size + slot = loc % index_page_size page_base = page * page_bytes offs = tl.arange(0, BLOCK_D) @@ -1490,7 +1488,7 @@ def _select_and_scatter_pool_slots_kernel( buf_ptr, cp_rank: tl.constexpr, payload_bytes: tl.constexpr, - slots_per_page: tl.constexpr, + index_page_size: tl.constexpr, head_dim: tl.constexpr, page_bytes: tl.constexpr, scale_region_off: tl.constexpr, @@ -1503,8 +1501,8 @@ def _select_and_scatter_pool_slots_kernel( return loc = tl.load(locs_ptr + row).to(tl.int64) - page = loc // slots_per_page - slot = loc % slots_per_page + page = loc // index_page_size + slot = loc % index_page_size page_base = page * page_bytes recv_row_base = (owner * n_total + row) * payload_bytes @@ -1526,7 +1524,7 @@ def all_gather_and_scatter_pool_slots( owner_rank: torch.Tensor, cp_size: int, cp_rank: int, - slots_per_page: int, + index_page_size: int, ) -> None: from sglang.srt.layers.dp_attention import attn_cp_all_gather_into_tensor @@ -1537,7 +1535,7 @@ def all_gather_and_scatter_pool_slots( head_dim = INDEX_HEAD_DIM payload_bytes = head_dim + 4 - scale_region_off = slots_per_page * head_dim + scale_region_off = index_page_size * head_dim page_bytes = buf.shape[1] device = buf.device @@ -1549,7 +1547,7 @@ def all_gather_and_scatter_pool_slots( local_locs, send_payload, payload_bytes=payload_bytes, - slots_per_page=slots_per_page, + index_page_size=index_page_size, head_dim=head_dim, page_bytes=page_bytes, scale_region_off=scale_region_off, @@ -1570,7 +1568,7 @@ def all_gather_and_scatter_pool_slots( buf, cp_rank=cp_rank, payload_bytes=payload_bytes, - slots_per_page=slots_per_page, + index_page_size=index_page_size, head_dim=head_dim, page_bytes=page_bytes, scale_region_off=scale_region_off, @@ -1605,7 +1603,7 @@ def _kpool_write_tail_and_maybe_compress_kernel( TAIL_SIZE: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_D: tl.constexpr, - SLOTS_PER_PAGE: tl.constexpr, + INDEX_PAGE_SIZE: tl.constexpr, BUF_NUMEL_PER_PAGE: tl.constexpr, S_OFFSET_NBYTES_IN_PAGE: tl.constexpr, PRESHUFFLE_TILE: tl.constexpr, @@ -1671,8 +1669,8 @@ def _kpool_write_tail_and_maybe_compress_kernel( quantized, scale = _hadamard_quantize_fp8(acc, denom, ROUND_SCALE) loc = tl.load(write_loc_ptr + b * write_loc_stride_0 + p) - loc_page_index = loc // SLOTS_PER_PAGE - loc_token_offset_in_page = loc % SLOTS_PER_PAGE + loc_page_index = loc // INDEX_PAGE_SIZE + loc_token_offset_in_page = loc % INDEX_PAGE_SIZE out_k_offsets = _kpool_cache_k_offsets( loc_page_index, loc_token_offset_in_page, @@ -1739,7 +1737,7 @@ def kpool_write_tail_and_maybe_compress( if effective_n_per_batch is not None: effective_n_per_batch = effective_n_per_batch.contiguous() - slots_per_page = pool.slots_per_page + index_page_size = pool.index_page_size buf_fp8 = buf.view(torch.float8_e4m3fn) buf_fp32 = buf.view(torch.float32) _kpool_write_tail_and_maybe_compress_kernel[(bs,)]( @@ -1767,10 +1765,10 @@ def kpool_write_tail_and_maybe_compress( TAIL_SIZE=tail_k.shape[1], HEAD_DIM=INDEX_HEAD_DIM, BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM), - SLOTS_PER_PAGE=slots_per_page, + INDEX_PAGE_SIZE=index_page_size, BUF_NUMEL_PER_PAGE=buf.shape[1], PRESHUFFLE_TILE=_preshuffle_tile(), - S_OFFSET_NBYTES_IN_PAGE=slots_per_page * pool.index_head_dim, + S_OFFSET_NBYTES_IN_PAGE=index_page_size * pool.index_head_dim, ROUND_SCALE=round_scale, HAS_EFFECTIVE_N=effective_n_per_batch is not None, MAX_CLOSED_POOLS=max_closed_pools, diff --git a/python/sglang/srt/layers/attention/dsa/kpool_plan.py b/python/sglang/srt/layers/attention/dsa/kpool_plan.py index 63bd8203b9..3902a0a840 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_plan.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_plan.py @@ -117,8 +117,8 @@ class KPoolWritePlan: effective_n_per_batch: Optional[torch.Tensor] = None -def _is_kpool_layout_enabled(pool_size: int, real_page_size: int) -> bool: - return pool_size > 1 and real_page_size == 64 and real_page_size % pool_size == 0 +def _is_kpool_layout_enabled(kpool: int, physical_page_size: int) -> bool: + return kpool > 1 and physical_page_size == 64 and physical_page_size % kpool == 0 @dataclass @@ -149,11 +149,11 @@ class _KPoolDecompose(NamedTuple): tail_n_write: int -def _decompose_compress(start: int, length: int, pool_size: int) -> _KPoolDecompose: - first_slot = start % pool_size - base_pool = start // pool_size - n_pool = (start + length) // pool_size - base_pool - consumed = max(0, n_pool * pool_size - first_slot) +def _decompose_compress(start: int, length: int, kpool: int) -> _KPoolDecompose: + first_slot = start % kpool + base_pool = start // kpool + n_pool = (start + length) // kpool - base_pool + consumed = max(0, n_pool * kpool - first_slot) tail_n = length - consumed return _KPoolDecompose( first_slot=first_slot, @@ -165,7 +165,7 @@ def _decompose_compress(start: int, length: int, pool_size: int) -> _KPoolDecomp def _append_compress_rows( plan: _KPoolCpuPlan, - pool_size: int, + kpool: int, batch_size: int, extend_seq_lens_cpu: List[int], seq_lens_cpu: List[int], @@ -178,7 +178,7 @@ def _append_compress_rows( seq_len = seq_lens_cpu[i] req = req_pool_indices_cpu[i] - d = _decompose_compress(seq_len - q_len, q_len, pool_size) + d = _decompose_compress(seq_len - q_len, q_len, kpool) if d.n_pool > 0: plan.pool_batch_idx.extend([i] * d.n_pool) @@ -186,16 +186,16 @@ def _append_compress_rows( plan.pool_pool_id.extend(range(d.base_pool, d.base_pool + d.n_pool)) plan.pool_n_from_tail.append(d.first_slot) plan.pool_n_from_tail.extend([0] * (d.n_pool - 1)) - bulk_start = q_offset + pool_size - d.first_slot + bulk_start = q_offset + kpool - d.first_slot plan.pool_chunk_src.append(q_offset) plan.pool_chunk_src.extend( - range(bulk_start, bulk_start + (d.n_pool - 1) * pool_size, pool_size) + range(bulk_start, bulk_start + (d.n_pool - 1) * kpool, kpool) ) plan.pool_tail_logical_base.extend( range( - d.base_pool * pool_size, - (d.base_pool + d.n_pool) * pool_size, - pool_size, + d.base_pool * kpool, + (d.base_pool + d.n_pool) * kpool, + kpool, ) ) @@ -211,8 +211,8 @@ def _append_compress_rows( def _append_local_rows( plan: _KPoolCpuPlan, - pool_size: int, - slots_per_page: int, + kpool: int, + index_page_size: int, local_extend_seq_lens_cpu: List[int], local_seq_lens_cpu: List[int], ) -> None: @@ -222,8 +222,8 @@ def _append_local_rows( ): assert q_len > 0, f"local_extend_seq_lens_cpu has non-positive {q_len = }" plan.ragged_q_len.append(q_len) - pool_seq_len = seq_len // pool_size - pool_pages_i = (pool_seq_len + slots_per_page - 1) // slots_per_page + pool_seq_len = seq_len // kpool + pool_pages_i = (pool_seq_len + index_page_size - 1) // index_page_size plan.ragged_pool_pages.append(pool_pages_i) plan.cu_pages_excl.append(plan.total_pool_pages) plan.cu_q_len_excl.append(q_offset) @@ -233,8 +233,8 @@ def _append_local_rows( def _kpool_cpu_plan( forward_batch: ForwardBatch, - pool_size: int, - slots_per_page: int, + kpool: int, + index_page_size: int, *, local_extend_seq_lens_cpu: Optional[List[int]] = None, local_seq_lens_cpu: Optional[List[int]] = None, @@ -252,7 +252,7 @@ def _kpool_cpu_plan( _append_compress_rows( plan, - pool_size, + kpool, forward_batch.batch_size, extend_seq_lens_cpu, seq_lens_cpu, @@ -265,8 +265,8 @@ def _kpool_cpu_plan( _append_local_rows( plan, - pool_size, - slots_per_page, + kpool, + index_page_size, local_extend_seq_lens_cpu, local_seq_lens_cpu, ) @@ -280,8 +280,8 @@ def _kpool_plan_to_gpu( local_real_page_table: torch.Tensor, local_seqlens_expanded: torch.Tensor, local_req_pool_indices: torch.Tensor, - pool_size: int, - slots_per_page: int, + kpool: int, + index_page_size: int, topk_transform_method: TopkTransformMethod, ) -> KPoolExtendPlan: from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod @@ -292,7 +292,7 @@ def _kpool_plan_to_gpu( n_rag = len(cpu.ragged_q_len) total_pool_pages = cpu.total_pool_pages - ragged_total_k_rows = total_pool_pages * slots_per_page + ragged_total_k_rows = total_pool_pages * index_page_size need_paged = ( topk_transform_method == TopkTransformMethod.PAGED @@ -370,21 +370,21 @@ def _kpool_plan_to_gpu( if n_pool > 0: pool_page_group = torch.div( - pool_pool_id_t, slots_per_page, rounding_mode="floor" + pool_pool_id_t, index_page_size, rounding_mode="floor" ) - token_page_row = pool_page_group * pool_size + token_page_row = pool_page_group * kpool packed_page = ( full_real_page_table[pool_batch_idx_t, token_page_row].to(torch.int64) - // pool_size + // kpool ) - pool_write_locs = packed_page * slots_per_page + torch.remainder( - pool_pool_id_t, slots_per_page + pool_write_locs = packed_page * index_page_size + torch.remainder( + pool_pool_id_t, index_page_size ) else: pool_write_locs = torch.empty((0,), dtype=torch.int64, device=device) pooled_seq_lens_expanded = torch.div( - local_seqlens_expanded, pool_size, rounding_mode="floor" + local_seqlens_expanded, kpool, rounding_mode="floor" ).to(torch.int32) if n_rag > 0: @@ -399,10 +399,10 @@ def _kpool_plan_to_gpu( cu_q_len_excl=cu_q_len_excl_t, ragged_q_len=ragged_q_len_t, pooled_seq_lens_expanded=pooled_seq_lens_expanded, - slots_per_page=slots_per_page, + index_page_size=index_page_size, total_pool_pages=total_pool_pages, total_q=pooled_seq_lens_expanded.shape[0], - pool_size=pool_size, + kpool=kpool, ) else: empty_i32_dev = torch.empty((0,), dtype=torch.int32, device=device) @@ -487,9 +487,9 @@ def init_kpool_extend_metadata( metadata: DSAMetadata, forward_batch: ForwardBatch, *, - pool_size: int, - real_page_size: int, - slots_per_page: int, + kpool: int, + physical_page_size: int, + index_page_size: int, topk_transform_method: TopkTransformMethod, full_real_page_table: torch.Tensor, full_seqlens_expanded: torch.Tensor, @@ -502,7 +502,7 @@ def init_kpool_extend_metadata( mode = forward_batch.forward_mode is_extend_like = mode.is_extend_without_speculative() or mode.is_draft_extend_v2() if ( - not _is_kpool_layout_enabled(pool_size, real_page_size) + not _is_kpool_layout_enabled(kpool, physical_page_size) or not is_extend_like or forward_batch.extend_seq_lens_cpu is None or forward_batch.seq_lens_cpu is None @@ -518,8 +518,8 @@ def init_kpool_extend_metadata( cpu = _kpool_cpu_plan( forward_batch, - pool_size, - slots_per_page, + kpool, + index_page_size, local_extend_seq_lens_cpu=local_extend_seq_lens_cpu, local_seq_lens_cpu=local_seq_lens_cpu, ) @@ -530,8 +530,8 @@ def init_kpool_extend_metadata( local_real_page_table, local_seqlens_expanded, local_req_pool_indices, - pool_size, - slots_per_page, + kpool, + index_page_size, topk_transform_method, ) return dataclasses.replace(metadata, kpool_extend_plan=plan) @@ -558,7 +558,7 @@ def _get_deep_gemm(): def _compute_pool_schedule_metadata( pool_seqlens: torch.Tensor, *, - slots_per_page: int, + index_page_size: int, ) -> Optional[torch.Tensor]: if not is_cuda(): return None @@ -567,7 +567,7 @@ def _compute_pool_schedule_metadata( return None return deep_gemm.get_paged_mqa_logits_metadata( pool_seqlens.contiguous().view(-1, 1).clamp(min=1), - slots_per_page, + index_page_size, deep_gemm.get_num_sms(), ) @@ -577,42 +577,40 @@ def init_pooled_paged_mqa_metadata( seqlens_32: torch.Tensor, forward_mode: ForwardMode, *, - pool_size: int, - real_page_size: int, - slots_per_page: int, + kpool: int, + physical_page_size: int, + index_page_size: int, build_schedule_metadata: bool = True, ) -> DSAMetadata: if ( - not _is_kpool_layout_enabled(pool_size, real_page_size) + not _is_kpool_layout_enabled(kpool, physical_page_size) or not is_cuda() or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()) ): return metadata pooled_page_table = build_pooled_page_table_64( - metadata.real_page_table, pool_size + metadata.real_page_table, kpool ).contiguous() if forward_mode.is_target_verify(): return dataclasses.replace( metadata, - pooled_index_kpool=pool_size, + pooled_index_kpool=kpool, pooled_real_page_table=pooled_page_table, ) - pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to( - torch.int32 - ) + pool_seqlens = torch.div(seqlens_32, kpool, rounding_mode="floor").to(torch.int32) schedule = ( _compute_pool_schedule_metadata( pool_seqlens, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) if build_schedule_metadata else None ) return dataclasses.replace( metadata, - pooled_index_kpool=pool_size, + pooled_index_kpool=kpool, pooled_cache_seqlens_int32=pool_seqlens, pooled_real_page_table=pooled_page_table, pooled_paged_mqa_schedule_metadata=schedule, @@ -624,13 +622,13 @@ def update_pooled_paged_mqa_metadata( seqlens_32: torch.Tensor, forward_mode: ForwardMode, *, - pool_size: int, - real_page_size: int, - slots_per_page: int, + kpool: int, + physical_page_size: int, + index_page_size: int, build_schedule_metadata: bool = True, ) -> None: if ( - not _is_kpool_layout_enabled(pool_size, real_page_size) + not _is_kpool_layout_enabled(kpool, physical_page_size) or not is_cuda() or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()) ): @@ -638,25 +636,23 @@ def update_pooled_paged_mqa_metadata( if forward_mode.is_target_verify(): torch.floor_divide( - metadata.real_page_table[:, ::pool_size], - pool_size, + metadata.real_page_table[:, ::kpool], + kpool, out=metadata.pooled_real_page_table, ) return if ( - metadata.pooled_index_kpool != pool_size + metadata.pooled_index_kpool != kpool or metadata.pooled_cache_seqlens_int32 is None or metadata.pooled_real_page_table is None ): return - pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to( - torch.int32 - ) + pool_seqlens = torch.div(seqlens_32, kpool, rounding_mode="floor").to(torch.int32) metadata.pooled_cache_seqlens_int32[: pool_seqlens.shape[0]].copy_(pool_seqlens) pooled_page_table = build_pooled_page_table_64( - metadata.real_page_table, pool_size + metadata.real_page_table, kpool ).contiguous() metadata.pooled_real_page_table[ : pooled_page_table.shape[0], : pooled_page_table.shape[1] @@ -668,7 +664,7 @@ def update_pooled_paged_mqa_metadata( ): new_schedule = _compute_pool_schedule_metadata( metadata.pooled_cache_seqlens_int32, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) if new_schedule is not None: metadata.pooled_paged_mqa_schedule_metadata.copy_(new_schedule) @@ -678,12 +674,12 @@ def _alloc_kpool_write_plan_buffers( *, max_bs: int, num_draft_tokens: int, - pool_size: int, + kpool: int, device: torch.device, is_verify: bool, is_v2: bool = False, ) -> KPoolWritePlan: - max_closed_pools = kpool_max_closed_pools(num_draft_tokens, pool_size) + max_closed_pools = kpool_max_closed_pools(num_draft_tokens, kpool) verify_extras = {} if is_verify: n_rows = max_bs * num_draft_tokens @@ -711,22 +707,22 @@ def init_kpool_write_plan_capture( metadata: DSAMetadata, *, max_bs: int, - pool_size: int, - real_page_size: int, + kpool: int, + physical_page_size: int, num_draft_tokens: int, device: torch.device, is_verify: bool, - slots_per_page: int, + index_page_size: int, is_v2: bool = False, build_schedule_metadata: bool = True, ) -> DSAMetadata: - if not _is_kpool_layout_enabled(pool_size, real_page_size) or num_draft_tokens == 0: + if not _is_kpool_layout_enabled(kpool, physical_page_size) or num_draft_tokens == 0: return metadata plan = _alloc_kpool_write_plan_buffers( max_bs=max_bs, num_draft_tokens=num_draft_tokens, - pool_size=pool_size, + kpool=kpool, device=device, is_verify=is_verify, is_v2=is_v2, @@ -734,7 +730,7 @@ def init_kpool_write_plan_capture( if is_verify and build_schedule_metadata: schedule = _compute_pool_schedule_metadata( plan.pool_seqlens_per_q, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) plan = dataclasses.replace(plan, pool_schedule_metadata=schedule) return dataclasses.replace(metadata, kpool_write_plan=plan) @@ -746,14 +742,14 @@ def update_kpool_write_plan( write_start: torch.Tensor, req_pool_indices: torch.Tensor, real_page_table: torch.Tensor, - pool_size: int, - real_page_size: int, + kpool: int, + physical_page_size: int, num_draft_tokens: int, forward_mode: ForwardMode, - slots_per_page: int, + index_page_size: int, effective_n_per_batch: Optional[torch.Tensor] = None, ) -> None: - if not _is_kpool_layout_enabled(pool_size, real_page_size) or not ( + if not _is_kpool_layout_enabled(kpool, physical_page_size) or not ( is_cuda() or is_hip() ): return @@ -775,9 +771,9 @@ def update_kpool_write_plan( write_loc_out=plan.write_loc, pool_seqlens_per_q_out=plan.pool_seqlens_per_q, seqlens_per_q_out=plan.seqlens_per_q, - pool_size=pool_size, + kpool=kpool, num_draft_tokens=num_draft_tokens, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) if ( @@ -792,7 +788,7 @@ def update_kpool_write_plan( if plan.pool_schedule_metadata is not None: new_schedule = _compute_pool_schedule_metadata( plan.pool_seqlens_per_q, - slots_per_page=slots_per_page, + index_page_size=index_page_size, ) if new_schedule is not None: plan.pool_schedule_metadata.copy_(new_schedule) @@ -802,12 +798,12 @@ def init_kpool_write_plan( metadata: DSAMetadata, forward_batch: ForwardBatch, *, - pool_size: int, - real_page_size: int, + kpool: int, + physical_page_size: int, real_page_table: torch.Tensor, num_draft_tokens: int, write_start: torch.Tensor, - slots_per_page: int, + index_page_size: int, effective_n_per_batch: Optional[torch.Tensor] = None, build_schedule_metadata: bool = True, ) -> DSAMetadata: @@ -816,7 +812,7 @@ def init_kpool_write_plan( is_decode = forward_mode.is_decode_or_idle() is_v2 = forward_mode.is_draft_extend_v2() is_ring_write = is_verify or is_decode or is_v2 - if not _is_kpool_layout_enabled(pool_size, real_page_size) or not is_ring_write: + if not _is_kpool_layout_enabled(kpool, physical_page_size) or not is_ring_write: return metadata pool = getattr(forward_batch, "token_to_kv_pool", None) @@ -829,12 +825,12 @@ def init_kpool_write_plan( metadata = init_kpool_write_plan_capture( metadata, max_bs=forward_batch.seq_lens.shape[0], - pool_size=pool_size, - real_page_size=real_page_size, + kpool=kpool, + physical_page_size=physical_page_size, num_draft_tokens=num_draft_tokens, device=forward_batch.seq_lens.device, is_verify=is_verify or is_v2, - slots_per_page=slots_per_page, + index_page_size=index_page_size, is_v2=is_v2, build_schedule_metadata=build_schedule_metadata, ) @@ -843,11 +839,11 @@ def init_kpool_write_plan( write_start=write_start, req_pool_indices=forward_batch.req_pool_indices, real_page_table=real_page_table, - pool_size=pool_size, - real_page_size=real_page_size, + kpool=kpool, + physical_page_size=physical_page_size, num_draft_tokens=num_draft_tokens, forward_mode=forward_mode, - slots_per_page=slots_per_page, + index_page_size=index_page_size, effective_n_per_batch=effective_n_per_batch, ) return metadata diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 4794c46022..fa8fb3ca57 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -446,7 +446,7 @@ class DeepseekSparseAttnBackend( ) self.dsa_index_topk = get_dsa_index_topk(hf_config) self.dsa_index_kpool = get_dsa_index_kpool(hf_config) - self.real_page_size = model_runner.page_size // self.dsa_index_kpool + self.physical_page_size = model_runner.page_size // self.dsa_index_kpool self._init_kpool_metadata_fusion() self.max_context_len = model_runner.model_config.context_len self._memory_saver_adapter = TorchMemorySaverAdapter.create( @@ -938,7 +938,7 @@ class DeepseekSparseAttnBackend( return self.req_to_token.shape[1] def _transform_table_1_to_real(self, page_table: torch.Tensor) -> torch.Tensor: - page_size = self.real_page_size + page_size = self.physical_page_size if page_size == 1: return page_table max_seqlen_k = page_table.shape[1] @@ -1021,11 +1021,11 @@ class DeepseekSparseAttnBackend( use_kpool = self.dsa_index_kpool > 1 if use_kpool: assert ( - self.real_page_size == 64 - and self.real_page_size % self.dsa_index_kpool == 0 + self.physical_page_size == 64 + and self.physical_page_size % self.dsa_index_kpool == 0 ), ( f"kpool path requires page_size == 64 and page_size % pool_size == 0; " - f"got page_size={self.real_page_size}, pool_size={self.dsa_index_kpool}." + f"got page_size={self.physical_page_size}, pool_size={self.dsa_index_kpool}." ) kpool_inputs = _KPoolForwardInputs() @@ -1239,7 +1239,7 @@ class DeepseekSparseAttnBackend( ) metadata = DSAMetadata( - page_size=self.real_page_size, + page_size=self.physical_page_size, cache_seqlens_int32=cache_seqlens_int32, max_seq_len_q=max_seqlen_q, max_seq_len_k=max_seqlen_k, @@ -1382,7 +1382,7 @@ class DeepseekSparseAttnBackend( self.dsa_drop_wide_page_table = ( is_cuda() and not _is_hip - and self.real_page_size > 1 + and self.physical_page_size > 1 and self.hisparse_coordinator is None and not self.speculative_num_draft_tokens # kpool's PAGED fused-topk mapping still reads page_table_1. @@ -1435,7 +1435,8 @@ class DeepseekSparseAttnBackend( "real_page_table": ( torch.zeros( max_num_tokens, - (max_ctx_len + self.real_page_size - 1) // self.real_page_size, + (max_ctx_len + self.physical_page_size - 1) + // self.physical_page_size, dtype=torch.int32, device=self.device, ) @@ -1627,7 +1628,7 @@ class DeepseekSparseAttnBackend( ) metadata = DSAMetadata( - page_size=self.real_page_size, + page_size=self.physical_page_size, cache_seqlens_int32=cache_seqlens_int32, max_seq_len_q=max_seqlen_q, max_seq_len_k=max_seqlen_k, @@ -1711,7 +1712,7 @@ class DeepseekSparseAttnBackend( bs=bs, max_len=max_len, dsa_index_topk=self.dsa_index_topk, - real_page_size=self.real_page_size, + physical_page_size=self.physical_page_size, ) cache_seqlens = metadata.cache_seqlens_int32 dsa_cache_seqlens = metadata.dsa_cache_seqlens_int32 @@ -1766,7 +1767,7 @@ class DeepseekSparseAttnBackend( bs=bs, max_seqlen_k=max_seqlen_k, dsa_index_topk=self.dsa_index_topk, - real_page_size=self.real_page_size, + physical_page_size=self.physical_page_size, next_n=self.speculative_num_draft_tokens, paged_mqa_ctx_lens_2d=paged_mqa_ctx_lens_2d, ) @@ -1854,7 +1855,7 @@ class DeepseekSparseAttnBackend( total_len=total_extend_len, max_seqlen_k=max_seqlen_k, dsa_index_topk=self.dsa_index_topk, - real_page_size=self.real_page_size, + physical_page_size=self.physical_page_size, max_extend_len=self.speculative_num_draft_tokens, max_total_len=bs * self.speculative_num_draft_tokens, static_extend_len=True, @@ -1934,8 +1935,8 @@ class DeepseekSparseAttnBackend( ) # NOTE(dark): (dsa-) cu_seqlens_q is always arange, no need to copy - assert self.real_page_size == metadata.page_size - if self.real_page_size > 1: + assert self.physical_page_size == metadata.page_size + if self.physical_page_size > 1: if not used_fused_metadata_generation: real_table = self._transform_table_1_to_real(page_indices) new_rows = real_table.shape[0] @@ -3052,7 +3053,7 @@ class DeepseekSparseAttnBackend( indices=page_table_1, seq_lens=seq_lens, workspace_buffer=self.workspace_buffer, - page_size=self.real_page_size, + page_size=self.physical_page_size, kv_cache_dim=self.kv_cache_dim, qk_nope_head_dim=self.qk_nope_head_dim, kv_lora_rank=self.kv_lora_rank, @@ -3089,8 +3090,8 @@ class DeepseekSparseAttnBackend( else: q_input = q_all - kv_cache = kv_cache.view(-1, self.real_page_size, 1, self.kv_cache_dim) - assert self.real_page_size == 64, "only page size 64 is supported" + kv_cache = kv_cache.view(-1, self.physical_page_size, 1, self.kv_cache_dim) + assert self.physical_page_size == 64, "only page size 64 is supported" if not self.dsa_kv_cache_store_fp8: # inefficiently quantize the whole cache @@ -3331,7 +3332,7 @@ class DeepseekSparseAttnBackend( D_ckv = kv_cache.shape[-1] # kv_cache: (N_tokens, 1, D_ckv) from MLATokenToKVPool → (N_pages, page_size, D_ckv) - kv_paged = kv_cache.view(-1, self.real_page_size, D_ckv) + kv_paged = kv_cache.view(-1, self.physical_page_size, D_ckv) block_table = metadata.real_page_table # (B, max_pages), page-indexed int32 seq_lens_k = metadata.cache_seqlens_int32 # (B,) total KV lengths @@ -3607,7 +3608,9 @@ class DeepseekSparseAttnBackend( self.token_to_kv_pool.set_mla_kv_buffer(layer, cache_loc, k, k_rope) k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = k_cache.view(-1, self.real_page_size, self.kv_cache_dim).unsqueeze(1) + kv_cache = k_cache.view( + -1, self.physical_page_size, self.kv_cache_dim + ).unsqueeze(1) if merge_query: q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) @@ -3677,7 +3680,7 @@ class DeepseekSparseAttnBackend( multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size) q = q_all.view(batch_size, 1, num_heads, head_dim) - kv = kv_cache.view(-1, 1, self.real_page_size, self.kv_cache_dim) + kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim) block_tables = page_table_1.unsqueeze(1) seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens diff --git a/python/sglang/srt/mem_cache/index_key_cache.py b/python/sglang/srt/mem_cache/index_key_cache.py index 2d8482dac0..2b74bbcfcb 100644 --- a/python/sglang/srt/mem_cache/index_key_cache.py +++ b/python/sglang/srt/mem_cache/index_key_cache.py @@ -35,7 +35,7 @@ class IndexKeyCache: pool = self.pool return ( num_pages, - pool.slots_per_page + pool.index_page_size * (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4), ) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 1556eb64c3..412c11db64 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -4290,8 +4290,8 @@ class HybridLinearKVPool(KVCache): return getattr(self.full_kv_pool, "tail_extra_slots", 0) @property - def slots_per_page(self) -> int: - return getattr(self.full_kv_pool, "slots_per_page", self.page_size) + def index_page_size(self) -> int: + return getattr(self.full_kv_pool, "index_page_size", self.page_size) def get_kv_size_bytes(self): return self.full_kv_pool.get_kv_size_bytes() @@ -5153,7 +5153,7 @@ class DSATokenToKVPool(MLATokenToKVPool): assert self.page_size % index_kpool == 0, ( f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}" ) - self.slots_per_page = self.page_size // index_kpool + self.index_page_size = self.page_size // index_kpool if index_buf_size is None: index_buf_size = size self.index_buf_size = index_buf_size diff --git a/python/sglang/srt/mem_cache/pool_host/dsa.py b/python/sglang/srt/mem_cache/pool_host/dsa.py index 7ba4b01616..5f06aeea31 100644 --- a/python/sglang/srt/mem_cache/pool_host/dsa.py +++ b/python/sglang/srt/mem_cache/pool_host/dsa.py @@ -98,7 +98,7 @@ def make_dsa_indexer_pool_decl( pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER ) -> HostPoolDecl: """Index key buffers riding on the full-KV pages: indices and layout both follow KV.""" - index_page_bytes = pool.slots_per_page * dsa_indexer_bytes_per_token_per_layer( + index_page_bytes = pool.index_page_size * dsa_indexer_bytes_per_token_per_layer( index_head_dim=pool.index_head_dim, quant_block_size=pool.quant_block_size, ) diff --git a/python/sglang/test/kits/dsa_metadata_kit.py b/python/sglang/test/kits/dsa_metadata_kit.py index 45de957994..6843e7a39e 100644 --- a/python/sglang/test/kits/dsa_metadata_kit.py +++ b/python/sglang/test/kits/dsa_metadata_kit.py @@ -32,7 +32,7 @@ def make_backend(mode, seq, req, *, fusion=True): backend.device = torch.device("cuda") backend.device_sm_major = torch.cuda.get_device_capability()[0] backend.num_q_heads = 64 - backend.real_page_size = 64 + backend.physical_page_size = 64 backend.dsa_index_topk = TOPK backend.dsa_index_kpool = POOL backend.speculative_num_draft_tokens = NEXT_N @@ -40,7 +40,7 @@ def make_backend(mode, seq, req, *, fusion=True): backend.dsa_decode_impl = "fa3" backend.dsa_prefill_impl = "fa3" backend.enable_auto_select_prefill_impl = False - backend.token_to_kv_pool = SimpleNamespace(slots_per_page=64) + backend.token_to_kv_pool = SimpleNamespace(index_page_size=64) # Only attention-dispatch state is synthetic; every metadata kernel is real. backend._is_in_breakable_cuda_graph = lambda: False backend._is_in_tc_piecewise_cuda_graph = lambda: False diff --git a/test/registered/kernels/ops/attention/test_dsa_indexer.py b/test/registered/kernels/ops/attention/test_dsa_indexer.py index 254ad92ed7..7434f74ac1 100644 --- a/test/registered/kernels/ops/attention/test_dsa_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsa_indexer.py @@ -1235,7 +1235,7 @@ class TestDSAIndexer(CustomTestCase): with self.subTest(topk_backend=topk_backend.value): backend = object.__new__(DeepseekSparseAttnBackend) backend.device = self.device - backend.real_page_size = 64 + backend.physical_page_size = 64 backend.hisparse_coordinator = None backend.speculative_num_draft_tokens = 0 backend.use_fused_topk = True diff --git a/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py b/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py index 0210670279..3e1f427025 100644 --- a/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py +++ b/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py @@ -16,7 +16,7 @@ register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-l class TestKPoolMetadataFusion(CustomTestCase): def test_verify_replay_boundaries_and_request_remapping(self): device = "cuda" - bs, next_n, width, topk, pool_size = 4, 6, 16384, 2048, 4 + bs, next_n, width, topk, kpool = 4, 6, 16384, 2048, 4 seq = torch.tensor([1, 61, 2047, 8191], device=device, dtype=torch.int64) req = torch.tensor([3, 1, 6, 0], device=device, dtype=torch.int64) table = torch.arange(8 * width, device=device, dtype=torch.int32).view(8, width) @@ -44,9 +44,9 @@ class TestKPoolMetadataFusion(CustomTestCase): bs=bs, max_seqlen_k=width, dsa_index_topk=topk, - real_page_size=64, + physical_page_size=64, next_n=next_n, - index_kpool=pool_size, + index_kpool=kpool, **buffers, ) @@ -65,7 +65,7 @@ class TestKPoolMetadataFusion(CustomTestCase): expanded = ( seq[:, None] + torch.arange(1, next_n + 1, device=device) ).flatten() - expected = torch.minimum(expanded, topk + expanded % pool_size).int() + expected = torch.minimum(expanded, topk + expanded % kpool).int() torch.testing.assert_close(buffers["seqlens_expanded"], expanded.int()) torch.testing.assert_close(buffers["dsa_cache_seqlens"], expected) torch.testing.assert_close( diff --git a/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py b/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py index 174b3a9008..a075c2b476 100644 --- a/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py +++ b/test/registered/kernels/ops/attention/test_dsa_kpool_multi_pool.py @@ -25,21 +25,21 @@ register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-l class TestDsaKpoolMultiPool(CustomTestCase): POOL_SIZE = 4 PAGE_SIZE = 256 - SLOTS_PER_PAGE = 64 + INDEX_PAGE_SIZE = 64 NUM_DRAFT_TOKENS = 6 def _pool(self) -> SimpleNamespace: return SimpleNamespace( page_size=self.PAGE_SIZE, index_head_dim=INDEX_HEAD_DIM, - slots_per_page=self.SLOTS_PER_PAGE, + index_page_size=self.INDEX_PAGE_SIZE, index_kpool=self.POOL_SIZE, tail_extra_slots=self.NUM_DRAFT_TOKENS, quant_block_size=128, ) def _empty_cache(self) -> torch.Tensor: - page_nbytes = self.SLOTS_PER_PAGE * INDEX_HEAD_DIM + self.SLOTS_PER_PAGE * 4 + page_nbytes = self.INDEX_PAGE_SIZE * INDEX_HEAD_DIM + self.INDEX_PAGE_SIZE * 4 return torch.zeros((1, page_nbytes), dtype=torch.uint8, device="cuda") def test_write_plan_records_every_candidate_pool(self): @@ -51,7 +51,7 @@ class TestDsaKpoolMultiPool(CustomTestCase): plan = _alloc_kpool_write_plan_buffers( max_bs=batch_size, num_draft_tokens=num_draft_tokens, - pool_size=self.POOL_SIZE, + kpool=self.POOL_SIZE, device=torch.device("cuda"), is_verify=True, ) @@ -79,9 +79,9 @@ class TestDsaKpoolMultiPool(CustomTestCase): write_loc_out=plan.write_loc, pool_seqlens_per_q_out=plan.pool_seqlens_per_q, seqlens_per_q_out=plan.seqlens_per_q, - pool_size=self.POOL_SIZE, + kpool=self.POOL_SIZE, num_draft_tokens=num_draft_tokens, - slots_per_page=self.SLOTS_PER_PAGE, + index_page_size=self.INDEX_PAGE_SIZE, ) torch.testing.assert_close(plan.req, req_pool_indices) @@ -94,10 +94,10 @@ class TestDsaKpoolMultiPool(CustomTestCase): plan.write_loc, torch.tensor( [ - [2 * self.SLOTS_PER_PAGE, 2 * self.SLOTS_PER_PAGE + 1], + [2 * self.INDEX_PAGE_SIZE, 2 * self.INDEX_PAGE_SIZE + 1], [ - 5 * self.SLOTS_PER_PAGE + 63, - 6 * self.SLOTS_PER_PAGE, + 5 * self.INDEX_PAGE_SIZE + 63, + 6 * self.INDEX_PAGE_SIZE, ], ], dtype=torch.int64, diff --git a/test/registered/kernels/ops/attention/test_dsa_metadata.py b/test/registered/kernels/ops/attention/test_dsa_metadata.py index 149e7c4e3c..db31b57309 100644 --- a/test/registered/kernels/ops/attention/test_dsa_metadata.py +++ b/test/registered/kernels/ops/attention/test_dsa_metadata.py @@ -27,10 +27,12 @@ def _dsa_seqlens(seqlens: torch.Tensor, topk: int) -> torch.Tensor: ) -def _real_page_table(page_table_1: torch.Tensor, real_page_size: int) -> torch.Tensor: - if real_page_size == 1: +def _real_page_table( + page_table_1: torch.Tensor, physical_page_size: int +) -> torch.Tensor: + if physical_page_size == 1: return page_table_1 - return page_table_1[:, ::real_page_size] // real_page_size + return page_table_1[:, ::physical_page_size] // physical_page_size def _make_req_to_token( @@ -59,7 +61,7 @@ class TestDSAMetadataKernels(CustomTestCase): *, max_len: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, ): bs = len(seq_lens_values) pool_size = max(bs + 3, 8) @@ -74,11 +76,11 @@ class TestDSAMetadataKernels(CustomTestCase): dsa_cu_seqlens_k = torch.empty(bs + 1, dtype=torch.int32, device=self.device) real_page_table = ( torch.empty( - (bs, (max_len + real_page_size - 1) // real_page_size), + (bs, (max_len + physical_page_size - 1) // physical_page_size), dtype=torch.int32, device=self.device, ) - if real_page_size > 1 + if physical_page_size > 1 else page_table_1 ) @@ -95,7 +97,7 @@ class TestDSAMetadataKernels(CustomTestCase): bs=bs, max_len=max_len, dsa_index_topk=dsa_index_topk, - real_page_size=real_page_size, + physical_page_size=physical_page_size, ) expected_cache = seq_lens.to(torch.int32) @@ -120,13 +122,13 @@ class TestDSAMetadataKernels(CustomTestCase): _assert_equal( dsa_cu_seqlens_k, _cu_seqlens(expected_dsa), "decode dsa_cu_seqlens_k" ) - if real_page_size > 1: + if physical_page_size > 1: real_width = real_page_table.shape[1] real_cols = torch.arange(real_width, dtype=torch.int32, device=self.device) real_live_mask = ( - real_cols.view(1, -1) * real_page_size + real_cols.view(1, -1) * physical_page_size ) < expected_cache.view(-1, 1) - expected_real = _real_page_table(expected_page_table, real_page_size) + expected_real = _real_page_table(expected_page_table, physical_page_size) _assert_equal( real_page_table[real_live_mask], expected_real[real_live_mask], @@ -139,7 +141,7 @@ class TestDSAMetadataKernels(CustomTestCase): *, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, next_n: int, fill_ctx_lens: bool, ): @@ -168,12 +170,12 @@ class TestDSAMetadataKernels(CustomTestCase): torch.empty( ( expanded_size, - (max_seqlen_k + real_page_size - 1) // real_page_size, + (max_seqlen_k + physical_page_size - 1) // physical_page_size, ), dtype=torch.int32, device=self.device, ) - if real_page_size > 1 + if physical_page_size > 1 else page_table_1 ) paged_mqa_ctx_lens_2d = ( @@ -196,7 +198,7 @@ class TestDSAMetadataKernels(CustomTestCase): bs=bs, max_seqlen_k=max_seqlen_k, dsa_index_topk=dsa_index_topk, - real_page_size=real_page_size, + physical_page_size=physical_page_size, next_n=next_n, paged_mqa_ctx_lens_2d=paged_mqa_ctx_lens_2d, ) @@ -231,13 +233,13 @@ class TestDSAMetadataKernels(CustomTestCase): _assert_equal( dsa_cu_seqlens_k, _cu_seqlens(expected_dsa), "target dsa_cu_seqlens_k" ) - if real_page_size > 1: + if physical_page_size > 1: real_width = real_page_table.shape[1] real_cols = torch.arange(real_width, dtype=torch.int32, device=self.device) real_live_mask = ( - real_cols.view(1, -1) * real_page_size + real_cols.view(1, -1) * physical_page_size ) < row_kv_lens.view(-1, 1) - expected_real = _real_page_table(expected_page_table, real_page_size) + expected_real = _real_page_table(expected_page_table, physical_page_size) _assert_equal( real_page_table[real_live_mask], expected_real[real_live_mask], @@ -256,7 +258,7 @@ class TestDSAMetadataKernels(CustomTestCase): *, max_seqlen_k: int, dsa_index_topk: int, - real_page_size: int, + physical_page_size: int, max_extend_len: int, max_total_len: int, static_extend_len: bool, @@ -287,11 +289,14 @@ class TestDSAMetadataKernels(CustomTestCase): ) real_page_table = ( torch.empty( - (max_total_len, (max_seqlen_k + real_page_size - 1) // real_page_size), + ( + max_total_len, + (max_seqlen_k + physical_page_size - 1) // physical_page_size, + ), dtype=torch.int32, device=self.device, ) - if real_page_size > 1 + if physical_page_size > 1 else page_table_1 ) @@ -311,7 +316,7 @@ class TestDSAMetadataKernels(CustomTestCase): total_len=total_len, max_seqlen_k=max_seqlen_k, dsa_index_topk=dsa_index_topk, - real_page_size=real_page_size, + physical_page_size=physical_page_size, max_extend_len=max_extend_len, max_total_len=max_total_len, static_extend_len=static_extend_len, @@ -365,14 +370,14 @@ class TestDSAMetadataKernels(CustomTestCase): _cu_seqlens(expected_dsa), "draft dsa_cu_seqlens_k", ) - if real_page_size > 1: - # Real-page column real_col maps to source column real_col*real_page_size. + if physical_page_size > 1: + # Real-page column real_col maps to source column real_col*physical_page_size. real_width = real_page_table.shape[1] real_cols = torch.arange(real_width, dtype=torch.int32, device=self.device) real_live_mask = ( - real_cols.view(1, -1) * real_page_size + real_cols.view(1, -1) * physical_page_size ) < row_kv_lens.view(-1, 1) - expected_real = _real_page_table(expected_page_table, real_page_size) + expected_real = _real_page_table(expected_page_table, physical_page_size) _assert_equal( real_page_table[:total_len][real_live_mask], expected_real[real_live_mask], @@ -380,25 +385,25 @@ class TestDSAMetadataKernels(CustomTestCase): ) def test_decode_matches_eager_reference(self): - for real_page_size in (1, 64): - with self.subTest(real_page_size=real_page_size): + for physical_page_size in (1, 64): + with self.subTest(physical_page_size=physical_page_size): self._check_decode( [1, 7, 65, 513], max_len=769, dsa_index_topk=64, - real_page_size=real_page_size, + physical_page_size=physical_page_size, ) def test_target_verify_matches_eager_reference(self): - for real_page_size, fill_ctx_lens in ((1, False), (64, True)): + for physical_page_size, fill_ctx_lens in ((1, False), (64, True)): with self.subTest( - real_page_size=real_page_size, fill_ctx_lens=fill_ctx_lens + physical_page_size=physical_page_size, fill_ctx_lens=fill_ctx_lens ): self._check_target_verify( [5, 63, 128], max_seqlen_k=257, dsa_index_topk=64, - real_page_size=real_page_size, + physical_page_size=physical_page_size, next_n=4, fill_ctx_lens=fill_ctx_lens, ) @@ -409,7 +414,7 @@ class TestDSAMetadataKernels(CustomTestCase): [4, 4, 4], max_seqlen_k=193, dsa_index_topk=64, - real_page_size=1, + physical_page_size=1, max_extend_len=4, max_total_len=12, static_extend_len=True, @@ -423,7 +428,7 @@ class TestDSAMetadataKernels(CustomTestCase): [3, 5, 2], max_seqlen_k=193, dsa_index_topk=64, - real_page_size=64, + physical_page_size=64, max_extend_len=5, max_total_len=10, static_extend_len=False, @@ -435,7 +440,7 @@ class TestDSAMetadataKernels(CustomTestCase): [3, 5, 2], max_seqlen_k=193, dsa_index_topk=64, - real_page_size=64, + physical_page_size=64, max_extend_len=5, max_total_len=16, static_extend_len=False, @@ -446,13 +451,13 @@ class TestDSAMetadataKernels(CustomTestCase): [], max_len=8, dsa_index_topk=64, - real_page_size=64, + physical_page_size=64, ) self._check_target_verify( [], max_seqlen_k=8, dsa_index_topk=64, - real_page_size=64, + physical_page_size=64, next_n=4, fill_ctx_lens=True, ) @@ -461,7 +466,7 @@ class TestDSAMetadataKernels(CustomTestCase): [], max_seqlen_k=8, dsa_index_topk=64, - real_page_size=64, + physical_page_size=64, max_extend_len=1, max_total_len=0, static_extend_len=True, @@ -473,13 +478,13 @@ class TestDSAMetadataKernels(CustomTestCase): [1_000_000, 999_983], max_len=max_len, dsa_index_topk=4096, - real_page_size=64, + physical_page_size=64, ) self._check_target_verify( [1_000_000], max_seqlen_k=max_len, dsa_index_topk=4096, - real_page_size=64, + physical_page_size=64, next_n=2, fill_ctx_lens=True, ) @@ -488,7 +493,7 @@ class TestDSAMetadataKernels(CustomTestCase): [4], max_seqlen_k=max_len, dsa_index_topk=4096, - real_page_size=64, + physical_page_size=64, max_extend_len=4, max_total_len=4, static_extend_len=True, @@ -501,13 +506,13 @@ class TestDSAMetadataKernels(CustomTestCase): seq_lens, max_len=1, dsa_index_topk=64, - real_page_size=1, + physical_page_size=1, ) self._check_target_verify( seq_lens, max_seqlen_k=1, dsa_index_topk=64, - real_page_size=1, + physical_page_size=1, next_n=1, fill_ctx_lens=False, ) @@ -516,7 +521,7 @@ class TestDSAMetadataKernels(CustomTestCase): [1] * bs, max_seqlen_k=1, dsa_index_topk=64, - real_page_size=1, + physical_page_size=1, max_extend_len=1, max_total_len=bs, static_extend_len=True, diff --git a/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py b/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py index 4902e296fa..ce8fbf5751 100644 --- a/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py +++ b/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py @@ -36,7 +36,7 @@ def _make_backend(*, allocate_counter: bool = True): backend = object.__new__(DeepseekSparseAttnBackend) backend.device = "cuda" backend.num_q_heads = _NUM_Q_HEADS - backend.real_page_size = 64 + backend.physical_page_size = 64 backend.hisparse_coordinator = None backend.speculative_num_draft_tokens = _NUM_DRAFT_TOKENS backend.dsa_index_kpool = 1 diff --git a/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py b/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py index 2bfeef8d1d..d36f29d617 100644 --- a/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py +++ b/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py @@ -43,7 +43,7 @@ class TestKpoolTopkTransformFused(CustomTestCase): def _expected_tokens( self, score_row: torch.Tensor, group_topk: int ) -> torch.Tensor: - """The pooled top-k groups expanded to their ``pool_size`` token ids.""" + """The pooled top-k groups expanded to their ``kpool`` token ids.""" groups = torch.topk(score_row.float().cpu(), group_topk).indices offsets = torch.arange(self.POOL_SIZE, dtype=torch.int64) return (groups.unsqueeze(1) * self.POOL_SIZE + offsets).reshape(-1) @@ -60,7 +60,7 @@ class TestKpoolTopkTransformFused(CustomTestCase): out = fast_kpool_topk_transform_fused( score=score, lengths=lengths, - pool_size=self.POOL_SIZE, + kpool=self.POOL_SIZE, topk=topk, seq_lens=seq_lens, ) @@ -82,6 +82,22 @@ class TestKpoolTopkTransformFused(CustomTestCase): score, out = self._run(rows=2, groups=1024, topk=2048) self._assert_pooled_columns(score, out, topk=2048) + def test_runtime_topk_matches_reference(self): + """Runtime dispatch must accept the same k-pool ratio as the fused kernel.""" + from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( + topk_from_pooled_history_logits, + ) + + score = self._distinct_scores(rows=2, groups=1024) + lengths = torch.full((2,), 1024, dtype=torch.int32, device="cuda") + out = topk_from_pooled_history_logits( + logits=score, + group_lengths=lengths, + kpool=self.POOL_SIZE, + topk=2048, + ) + self._assert_pooled_columns(score, out.cpu(), topk=2048) + def test_group_topk_128_matches_reference(self): score, out = self._run(rows=2, groups=512, topk=512) self._assert_pooled_columns(score, out, topk=512) @@ -118,7 +134,7 @@ class TestKpoolTopkTransformFused(CustomTestCase): @pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires a GPU") def test_topk_membership_including_overfull_coarse_bins(distribution, start): torch.manual_seed(1234) - rows, width, length, group_topk, pool_size = 4, 50000, 32768, 512, 4 + rows, width, length, group_topk, kpool = 4, 50000, 32768, 512, 4 scores = torch.randn(rows, width, device="cuda") if distribution == "equal": scores.fill_(1) @@ -130,18 +146,18 @@ def test_topk_membership_including_overfull_coarse_bins(distribution, start): result = fast_kpool_topk_transform_fused( scores, lengths, - pool_size, - group_topk * pool_size, + kpool, + group_topk * kpool, row_starts=starts, - seq_lens=lengths * pool_size + 3, + seq_lens=lengths * kpool + 3, ) - groups = result[:, :2048:pool_size].long() // pool_size + groups = result[:, :2048:kpool].long() // kpool assert bool(((groups >= 0) & (groups < length)).all()) for row in groups: assert torch.unique(row).numel() == group_topk torch.testing.assert_close( - result[:, :2048].reshape(rows, group_topk, pool_size).long(), - groups.unsqueeze(-1) * pool_size + torch.arange(pool_size, device="cuda"), + result[:, :2048].reshape(rows, group_topk, kpool).long(), + groups.unsqueeze(-1) * kpool + torch.arange(kpool, device="cuda"), atol=0, rtol=0, ) @@ -154,7 +170,7 @@ def test_topk_membership_including_overfull_coarse_bins(distribution, start): torch.testing.assert_close( result[:, -3:], torch.arange( - length * pool_size, length * pool_size + 3, dtype=torch.int32, device="cuda" + length * kpool, length * kpool + 3, dtype=torch.int32, device="cuda" ).expand(rows, -1), atol=0, rtol=0, diff --git a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py index c17cec7c40..bf38ba47e1 100644 --- a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py +++ b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py @@ -94,7 +94,7 @@ class TestDSABackendDPPadding(unittest.TestCase): token_to_kv_pool=SimpleNamespace( get_key_buffer=lambda _layer_id: torch.zeros((24, 3)) ), - real_page_size=1, + physical_page_size=1, kv_cache_dim=3, use_fused_topk=False, qk_nope_head_dim=2, diff --git a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py index 7905098816..454c861c7a 100644 --- a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py +++ b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py @@ -24,7 +24,7 @@ class TestKpoolPreshuffledCacheLayout(CustomTestCase): """A layout mismatch between the two writers never raises; it silently returns the wrong top-k.""" torch.manual_seed(0) - pool = SimpleNamespace(page_size=4 * PAGE_SIZE, slots_per_page=PAGE_SIZE) + pool = SimpleNamespace(page_size=4 * PAGE_SIZE, index_page_size=PAGE_SIZE) num_pages, seq_len = 2, PAGE_SIZE + 5 buf = torch.zeros( num_pages, diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index ca8b05e26e..c059fda4cf 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -691,7 +691,7 @@ class TestKPoolPerRequestChunking(CustomTestCase): ) with _chunking_patches( rows_per_chunk=rows_per_chunk, - kv_pool=SimpleNamespace(page_size=256, slots_per_page=64), + kv_pool=SimpleNamespace(page_size=256, index_page_size=64), ): result = IndexerKPool._get_topk_ragged_kpool( backend, diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index fe332aa246..1ba44f9bbd 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -270,7 +270,7 @@ class TestDraftSidecarPoolDispatch(CustomTestCase): draft_kv_pool.size = 800 draft_kv_pool.index_head_dim = 128 draft_kv_pool.page_size = 64 - draft_kv_pool.slots_per_page = 64 + draft_kv_pool.index_page_size = 64 draft_kv_pool.skip_topk_layers = [False] draft_kv_pool.index_key_cache = SimpleNamespace(buffer=[object()]) draft_host_pool = SimpleNamespace(layer_num=1) @@ -337,7 +337,7 @@ def _dsa_pool_stub(*, layer_num: int, size: int = 4096, shard: tuple | None = No pool.kv_cache_dim = 576 pool.index_head_dim = 128 pool.page_size = 64 - pool.slots_per_page = 64 + pool.index_page_size = 64 pool.skip_topk_layers = [False] * layer_num pool.index_key_cache = SimpleNamespace(buffer=[object()] * layer_num) pool.layer_shard_enabled = shard is not None diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 88c66e2fba..005b771232 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -484,7 +484,7 @@ class TestDSAIndexerPoolDecl(CustomTestCase): index_head_dim=128, quant_block_size=128, page_size=64, - slots_per_page=64, + index_page_size=64, skip_topk_layers=[False] * 5, ) diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index 0862247e11..6970608de0 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -84,7 +84,7 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): index_head_dim=8, quant_block_size=4, page_size=2, - slots_per_page=2, + index_page_size=2, skip_topk_layers=[False] * 2, ) indexer_host = DSAIndexerPoolHost(