[DSA] k-pool: 256-token logical page so index page id = logical page id (#42178)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
Khoa Pham
2026-10-05 15:02:44 -07:00
committed by GitHub
co-authored by Cursor Ke Bao
parent c765f8818a
commit 61ba98443d
17 changed files with 122 additions and 74 deletions
@@ -41,8 +41,9 @@ class GetK:
def slow(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
num_pages = (seq_len + pool.page_size - 1) // pool.page_size
seq_len_ = num_pages * pool.page_size
page_size = pool.slots_per_page
num_pages = (seq_len + page_size - 1) // page_size
seq_len_ = num_pages * page_size
index_k_fp8 = torch.empty(
(seq_len_, pool.index_head_dim),
dtype=torch.uint8,
@@ -50,9 +51,9 @@ class GetK:
)
for i in range(num_pages):
page_index = page_indices[i]
index_k_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[
page_index
][: pool.page_size * pool.index_head_dim].view(-1, pool.index_head_dim)
index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
: page_size * pool.index_head_dim
].view(-1, pool.index_head_dim)
return index_k_fp8[:seq_len]
@@ -70,7 +71,7 @@ class GetK:
# page_indices: (num_pages,), element := a page index
buf_numel_per_page = buf.shape[1]
num_k_bytes_per_page = pool.page_size * pool.index_head_dim
num_k_bytes_per_page = pool.slots_per_page * pool.index_head_dim
num_k_bytes_per_token = pool.index_head_dim
# buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8
@@ -99,7 +100,7 @@ class GetK:
buf=buf,
page_indices=page_indices,
seq_len=seq_len,
page_size=pool.page_size,
page_size=pool.slots_per_page,
index_head_dim=pool.index_head_dim,
)
@@ -113,8 +114,9 @@ class GetS:
def slow(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
num_pages = (seq_len + pool.page_size - 1) // pool.page_size
seq_len_ = num_pages * pool.page_size
page_size = pool.slots_per_page
num_pages = (seq_len + page_size - 1) // page_size
seq_len_ = num_pages * page_size
assert pool.index_head_dim // pool.quant_block_size == 1
index_k_scale_fp8 = torch.empty(
(seq_len_, 4),
@@ -123,9 +125,9 @@ class GetS:
)
for i in range(num_pages):
page_index = page_indices[i]
index_k_scale_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[
page_index
][pool.page_size * pool.index_head_dim :].view(-1, 4)
index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
page_size * pool.index_head_dim :
].view(-1, 4)
return index_k_scale_fp8[:seq_len]
@classmethod
@@ -138,9 +140,9 @@ class GetS:
"""
buf_numel_per_page = buf.shape[1]
num_s_bytes_per_page = buf.shape[1] - pool.page_size * pool.index_head_dim
s_offset_in_page = pool.slots_per_page * pool.index_head_dim
num_s_bytes_per_page = buf.shape[1] - s_offset_in_page
num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4
s_offset_in_page = pool.page_size * pool.index_head_dim
flat_buf = buf.flatten()
flat_indices = (
@@ -168,7 +170,7 @@ class GetS:
buf=buf,
page_indices=page_indices,
seq_len=seq_len,
page_size=pool.page_size,
page_size=pool.slots_per_page,
index_head_dim=pool.index_head_dim,
)
@@ -196,7 +198,7 @@ class GetKAndS:
):
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
page_size = pool.page_size
page_size = pool.slots_per_page
index_head_dim = pool.index_head_dim
quant_block_size = pool.quant_block_size
scale_elems = index_head_dim // quant_block_size
@@ -252,7 +254,7 @@ class GetKAndS:
seq_lens=seq_len_tensor,
seq_len_sum=seq_len_sum,
max_seq_len=max_seq_len,
page_size=pool.page_size,
page_size=pool.slots_per_page,
index_head_dim=pool.index_head_dim,
)
@@ -271,7 +273,7 @@ class SetKAndS:
loc=loc,
index_k=index_k,
index_k_scale=index_k_scale,
page_size=pool.page_size,
page_size=pool.slots_per_page,
)
@@ -33,6 +33,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"""Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook."""
cfg = resolving_view(server_args)
from sglang.srt.configs.model_config import (
get_dsa_index_kpool,
is_deepseek_dsa,
unwrap_modelopt_quantization_config,
)
@@ -139,8 +140,16 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"needs Triton>=3.5.0 or AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1)."
)
else:
overrides["page_size"] = 64
logger.warning("Setting page size to 64 for DeepSeek DSA.")
index_kpool = get_dsa_index_kpool(hf_config)
if index_kpool > 1 and getattr(cfg, "dcp_size", 1) > 1:
raise ValueError(
"--dcp-size > 1 is not supported for DSA with "
f"index_kpool={index_kpool}: a pooled index key spans "
"positions owned by different DCP ranks."
)
page_size = 64 * index_kpool
overrides["page_size"] = page_size
logger.warning(f"Setting page size to {page_size} for DeepSeek DSA.")
elif get_platform().is_xpu:
overrides["page_size"] = 128
logger.warning("Setting page size to 128 for DeepSeek DSA on XPU.")
@@ -68,7 +68,7 @@ class DeepseekSparseAttnBackendKPoolMixin:
return dsa_impl
def _kpool_slots_per_page(self) -> int:
return getattr(self.token_to_kv_pool, "slots_per_page", self.real_page_size)
return self.token_to_kv_pool.slots_per_page
def _build_kpool_paged_mqa_schedule_metadata(self) -> bool:
if self.device_sm_major == 9:
@@ -523,7 +523,7 @@ class IndexerKPool(MultiPlatformOp):
)
pool_start_id = first_pos // kpool
page_size = get_token_to_kv_pool().page_size
page_size = attn_metadata.page_size
use_returned_compressed = return_compressed and (
not write_cache or pool_start_id == 0
)
@@ -955,7 +955,7 @@ class IndexerKPool(MultiPlatformOp):
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
pool = get_token_to_kv_pool()
page_size = pool.page_size
page_size = pool.slots_per_page
# DeepGEMM paged-MQA requires 64-token pages.
assert page_size == 64, "only support page size 64"
@@ -1241,7 +1241,8 @@ class IndexerKPool(MultiPlatformOp):
)
pool_size = self.index_kpool
page_size = get_token_to_kv_pool().page_size
page_size = metadata.attn_metadata.page_size
slots_per_page = get_token_to_kv_pool().slots_per_page
token_nums = q_fp8.shape[0]
tail_pool = pool_size - 1
topk_result = torch.empty(
@@ -1355,7 +1356,9 @@ class IndexerKPool(MultiPlatformOp):
token_page_table = block_tables[
i, :num_token_pages
].contiguous()
pool_pages = (curr_pool_start + page_size - 1) // page_size
pool_pages = (
curr_pool_start + slots_per_page - 1
) // slots_per_page
pooled_page_table = build_pooled_page_table_64(
token_page_table, pool_size
)[:pool_pages].contiguous()
@@ -1395,7 +1398,9 @@ class IndexerKPool(MultiPlatformOp):
token_page_table = block_tables[
i, :num_token_pages
].contiguous()
pool_pages = (pool_seq_len + page_size - 1) // page_size
pool_pages = (
pool_seq_len + slots_per_page - 1
) // slots_per_page
pooled_page_table = build_pooled_page_table_64(
token_page_table, pool_size
)[:pool_pages].contiguous()
@@ -1494,7 +1499,7 @@ class IndexerKPool(MultiPlatformOp):
assert forward_batch.forward_mode.is_extend_without_speculative()
page_size = get_token_to_kv_pool().page_size
page_size = get_token_to_kv_pool().slots_per_page
assert page_size == 64, "only support page size 64"
assert len(weights.shape) == 3
weights = weights.squeeze(-1)
@@ -64,7 +64,7 @@ def build_pooled_page_table_64(
idx = torch.arange(
0, page_table_64.shape[-1], pool_size, device=page_table_64.device
)
return page_table_64[..., idx]
return page_table_64[..., idx] // pool_size
def gather_index_k_scale_prefix_into(
@@ -79,7 +79,7 @@ def gather_index_k_scale_prefix_into(
assert page_indices.dtype in (torch.int32, torch.int64)
assert k_out.dtype == torch.uint8
assert scale_out.dtype == torch.float32
assert pool.page_size == BLOCK_SIZE_K
assert pool.slots_per_page == BLOCK_SIZE_K
assert k_out.shape[0] >= seq_len
assert k_out.shape[1] == INDEX_HEAD_DIM
assert scale_out.shape[0] >= seq_len
@@ -96,11 +96,11 @@ def gather_index_k_scale_prefix_into(
page_indices,
k_out,
scale_out,
PAGE_SIZE=pool.page_size,
PAGE_SIZE=pool.slots_per_page,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
HEAD_DIM=INDEX_HEAD_DIM,
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM,
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * INDEX_HEAD_DIM,
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
)
@@ -159,8 +159,6 @@ def kpool_build_ragged_layout(
total_q: int,
pool_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# One packed-cache page represents 64 pools, so its source is every
# pool_size-th real token page.
device = full_page_table.device
n_rag = cu_pages_excl.shape[0]
concat_page_table = torch.empty(
@@ -224,7 +222,11 @@ def _kpool_build_ragged_layout_kernel(
mask=p_mask,
other=0,
)
tl.store(concat_page_table_ptr + page_start + p_offs, pages, mask=p_mask)
tl.store(
concat_page_table_ptr + page_start + p_offs,
pages // POOL_SIZE,
mask=p_mask,
)
for q_off in tl.range(0, BLOCK_Q * tl.cdiv(q_count, BLOCK_Q), BLOCK_Q):
q_offs = q_off + tl.arange(0, BLOCK_Q)
@@ -377,11 +379,14 @@ def _update_kpool_write_plan_kernel(
)
# Promote the row term before multiplication; 1M-context strides
# overflow int32 around row 2048.
packed_page = tl.load(
real_page_table_ptr
+ (b * N).to(tl.int64) * real_page_table_stride_0
+ token_page_row
).to(tl.int64)
packed_page = (
tl.load(
real_page_table_ptr
+ (b * N).to(tl.int64) * real_page_table_stride_0
+ token_page_row
).to(tl.int64)
// POOL_SIZE
)
write_loc = packed_page * SLOTS_PER_PAGE + (pool_id % SLOTS_PER_PAGE)
tl.store(
write_loc_out_ptr + b * write_loc_out_stride_0 + p,
@@ -398,10 +403,11 @@ def compute_pooled_write_locs(
pool_ids = pool_ids.to(torch.int64)
pool_page_group = torch.div(pool_ids, BLOCK_SIZE_K, rounding_mode="floor")
token_page_row = pool_page_group * pool_size
packed_page = page_table_64.index_select(0, token_page_row.to(torch.int64))
return packed_page.to(torch.int64) * BLOCK_SIZE_K + torch.remainder(
pool_ids, BLOCK_SIZE_K
packed_page = (
page_table_64.index_select(0, token_page_row.to(torch.int64)).to(torch.int64)
// pool_size
)
return packed_page * BLOCK_SIZE_K + torch.remainder(pool_ids, BLOCK_SIZE_K)
def history_group_budget_for_topk(topk: int, pool_size: int) -> int:
@@ -712,7 +718,7 @@ def kpool_softmax_rotate_write_cache(
assert slot_score.dtype in KPOOL_SCORE_DTYPES
assert ape.dtype == torch.float32
assert buf.dtype == torch.uint8
assert pool.page_size == BLOCK_SIZE_K
assert pool.slots_per_page == BLOCK_SIZE_K
assert pool.index_head_dim == INDEX_HEAD_DIM
assert loc.dtype == torch.int64
assert write_cache or return_compressed
@@ -771,12 +777,12 @@ def kpool_softmax_rotate_write_cache(
slot_score.stride(0),
slot_score.stride(1),
ape.stride(0),
PAGE_SIZE=pool.page_size,
PAGE_SIZE=pool.slots_per_page,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
POOL_SIZE=slot_k.shape[1],
HEAD_DIM=slot_k.shape[2],
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim,
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim,
ROUND_SCALE=round_scale,
HAS_WRITE_MASK=has_write_mask,
RETURN_COMPRESSED=return_compressed,
@@ -816,7 +822,7 @@ def kpool_decode_update_and_maybe_write_cache(
assert slot_score.dtype == tail_score.dtype
assert ape.dtype == torch.float32
assert buf.dtype == torch.uint8
assert pool.page_size == BLOCK_SIZE_K
assert pool.slots_per_page == BLOCK_SIZE_K
assert pool.index_head_dim == INDEX_HEAD_DIM
assert tail_k.is_contiguous()
assert tail_score.is_contiguous()
@@ -1165,7 +1171,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
+ row * block_tables_stride_0
+ token_page_row * block_tables_stride_1,
)
loc_page_index = packed_page.to(tl.int64)
loc_page_index = packed_page.to(tl.int64) // POOL_SIZE
loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE
out_k_offsets = _kpool_cache_k_offsets(
loc_page_index,
@@ -373,8 +373,9 @@ def _kpool_plan_to_gpu(
pool_pool_id_t, slots_per_page, rounding_mode="floor"
)
token_page_row = pool_page_group * pool_size
packed_page = full_real_page_table[pool_batch_idx_t, token_page_row].to(
torch.int64
packed_page = (
full_real_page_table[pool_batch_idx_t, token_page_row].to(torch.int64)
// pool_size
)
pool_write_locs = packed_page * slots_per_page + torch.remainder(
pool_pool_id_t, slots_per_page
@@ -369,7 +369,6 @@ class DeepseekSparseAttnBackend(
self.forward_metadata: DSAMetadata
self.device = model_runner.device
assert isinstance(model_runner.page_size, int)
self.real_page_size = model_runner.page_size
self.num_splits = (
1 if get_exec().deterministic.enable_deterministic_inference else 0
)
@@ -381,6 +380,7 @@ class DeepseekSparseAttnBackend(
)
self.dsa_index_topk = get_dsa_index_topk(hf_config)
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
self.real_page_size = model_runner.page_size // self.dsa_index_kpool
self._init_kpool_metadata_fusion()
self.max_context_len = model_runner.model_config.context_len
self._memory_saver_adapter = TorchMemorySaverAdapter.create(
@@ -35,7 +35,7 @@ class IndexKeyCache:
pool = self.pool
return (
num_pages,
pool.page_size
pool.slots_per_page
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
)
+15 -8
View File
@@ -5146,7 +5146,10 @@ class DSATokenToKVPool(MLATokenToKVPool):
self.index_kpool = index_kpool
self.index_kpool_compress = index_kpool_compress
self.tail_extra_slots = tail_extra_slots
self.slots_per_page = self.page_size
assert self.page_size % index_kpool == 0, (
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
)
self.slots_per_page = self.page_size // index_kpool
if index_buf_size is None:
index_buf_size = size
self.index_buf_size = index_buf_size
@@ -5160,22 +5163,26 @@ class DSATokenToKVPool(MLATokenToKVPool):
)
assert len(self.skip_topk_layers) == layer_num
physical_page_size = self.page_size // index_kpool
if _is_hip:
if aiter_can_use_preshuffle_paged_mqa():
assert self.page_size % 16 == 0, (
f"HIP preshuffle requires page_size to be a multiple of 16, got {self.page_size}"
assert physical_page_size % 16 == 0, (
f"HIP preshuffle requires page_size to be a multiple of 16, got {physical_page_size}"
)
else:
assert self.page_size == 1, (
f"HIP legacy DSA path requires page_size == 1, got {self.page_size}"
assert physical_page_size == 1, (
f"HIP legacy DSA path requires page_size == 1, got {physical_page_size}"
)
elif is_xpu():
assert self.page_size in (
assert physical_page_size in (
64,
128,
), f"XPU DSA requires page_size 64 or 128, got {self.page_size}"
), f"XPU DSA requires page_size 64 or 128, got {physical_page_size}"
else:
assert self.page_size == 64
assert physical_page_size == 64, (
f"DSA requires 64-token physical pages, got page_size={self.page_size} "
f"with index_kpool={index_kpool}"
)
self.index_key_cache = self._create_index_key_cache()
self._init_kpool_compress_tail_buffers(
index_kpool=index_kpool,
+9 -4
View File
@@ -98,16 +98,21 @@ def make_dsa_indexer_pool_decl(
pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER
) -> HostPoolDecl:
"""Index key buffers riding on the full-KV pages: indices and layout both follow KV."""
index_page_bytes = pool.slots_per_page * dsa_indexer_bytes_per_token_per_layer(
index_head_dim=pool.index_head_dim,
quant_block_size=pool.quant_block_size,
)
assert index_page_bytes % pool.page_size == 0, (
f"index page of {index_page_bytes} bytes does not split over "
f"page_size={pool.page_size} tokens"
)
return HostPoolDecl(
pool_name=name,
device_pool=pool,
indices_from_pool=PoolName.KV,
layout_source=PoolName.KV,
storage_info=HostPoolStorageInfo(
bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer(
index_head_dim=pool.index_head_dim,
quant_block_size=pool.quant_block_size,
),
bytes_per_token_per_layer=index_page_bytes // pool.page_size,
dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype,
),
host_pool_builder=DSAIndexerHostPoolBuilder(),
@@ -21,6 +21,7 @@ from sglang.srt.configs.model_config import (
AttentionArch,
dsa_layer_skips_topk,
get_dsa_index_head_dim,
get_dsa_index_kpool,
get_minimax_sparse_attention_config,
get_minimax_sparse_disable_value_layer_ids,
get_minimax_sparse_layer_ids,
@@ -515,8 +516,9 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
allocate_all_layers: bool = False,
) -> int:
index_head_dim = get_dsa_index_head_dim(kvc.model_config.hf_config)
indexer_size_per_token = (
index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4
indexer_size_per_token = ceil_div(
index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4,
get_dsa_index_kpool(kvc.model_config.hf_config),
)
element_size = torch._utils._element_size(
DSATokenToKVPool.index_k_with_scale_buffer_dtype
@@ -642,9 +644,10 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
if is_deepseek_dsa(model_config.hf_config):
index_head_dim = get_dsa_index_head_dim(model_config.hf_config)
index_elements = (
index_elements = ceil_div(
index_head_dim
+ index_head_dim // DSATokenToKVPool.quant_block_size * 4
+ index_head_dim // DSATokenToKVPool.quant_block_size * 4,
get_dsa_index_kpool(model_config.hf_config),
)
self._full_per_token += index_elements * torch._utils._element_size(
DSATokenToKVPool.index_k_with_scale_buffer_dtype