[DSA] k-pool: 256-token logical page so index page id = logical page id (#42178)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
co-authored by
Cursor
Ke Bao
parent
c765f8818a
commit
61ba98443d
@@ -41,8 +41,9 @@ class GetK:
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
num_pages = (seq_len + pool.page_size - 1) // pool.page_size
|
||||
seq_len_ = num_pages * pool.page_size
|
||||
page_size = pool.slots_per_page
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
index_k_fp8 = torch.empty(
|
||||
(seq_len_, pool.index_head_dim),
|
||||
dtype=torch.uint8,
|
||||
@@ -50,9 +51,9 @@ class GetK:
|
||||
)
|
||||
for i in range(num_pages):
|
||||
page_index = page_indices[i]
|
||||
index_k_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[
|
||||
page_index
|
||||
][: pool.page_size * pool.index_head_dim].view(-1, pool.index_head_dim)
|
||||
index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
|
||||
: page_size * pool.index_head_dim
|
||||
].view(-1, pool.index_head_dim)
|
||||
|
||||
return index_k_fp8[:seq_len]
|
||||
|
||||
@@ -70,7 +71,7 @@ class GetK:
|
||||
# page_indices: (num_pages,), element := a page index
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
num_k_bytes_per_page = pool.page_size * pool.index_head_dim
|
||||
num_k_bytes_per_page = pool.slots_per_page * pool.index_head_dim
|
||||
num_k_bytes_per_token = pool.index_head_dim
|
||||
|
||||
# buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8
|
||||
@@ -99,7 +100,7 @@ class GetK:
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.page_size,
|
||||
page_size=pool.slots_per_page,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -113,8 +114,9 @@ class GetS:
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
num_pages = (seq_len + pool.page_size - 1) // pool.page_size
|
||||
seq_len_ = num_pages * pool.page_size
|
||||
page_size = pool.slots_per_page
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
assert pool.index_head_dim // pool.quant_block_size == 1
|
||||
index_k_scale_fp8 = torch.empty(
|
||||
(seq_len_, 4),
|
||||
@@ -123,9 +125,9 @@ class GetS:
|
||||
)
|
||||
for i in range(num_pages):
|
||||
page_index = page_indices[i]
|
||||
index_k_scale_fp8[i * pool.page_size : (i + 1) * pool.page_size] = buf[
|
||||
page_index
|
||||
][pool.page_size * pool.index_head_dim :].view(-1, 4)
|
||||
index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
|
||||
page_size * pool.index_head_dim :
|
||||
].view(-1, 4)
|
||||
return index_k_scale_fp8[:seq_len]
|
||||
|
||||
@classmethod
|
||||
@@ -138,9 +140,9 @@ class GetS:
|
||||
"""
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
num_s_bytes_per_page = buf.shape[1] - pool.page_size * pool.index_head_dim
|
||||
s_offset_in_page = pool.slots_per_page * pool.index_head_dim
|
||||
num_s_bytes_per_page = buf.shape[1] - s_offset_in_page
|
||||
num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4
|
||||
s_offset_in_page = pool.page_size * pool.index_head_dim
|
||||
|
||||
flat_buf = buf.flatten()
|
||||
flat_indices = (
|
||||
@@ -168,7 +170,7 @@ class GetS:
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.page_size,
|
||||
page_size=pool.slots_per_page,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -196,7 +198,7 @@ class GetKAndS:
|
||||
):
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
|
||||
|
||||
page_size = pool.page_size
|
||||
page_size = pool.slots_per_page
|
||||
index_head_dim = pool.index_head_dim
|
||||
quant_block_size = pool.quant_block_size
|
||||
scale_elems = index_head_dim // quant_block_size
|
||||
@@ -252,7 +254,7 @@ class GetKAndS:
|
||||
seq_lens=seq_len_tensor,
|
||||
seq_len_sum=seq_len_sum,
|
||||
max_seq_len=max_seq_len,
|
||||
page_size=pool.page_size,
|
||||
page_size=pool.slots_per_page,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -271,7 +273,7 @@ class SetKAndS:
|
||||
loc=loc,
|
||||
index_k=index_k,
|
||||
index_k_scale=index_k_scale,
|
||||
page_size=pool.page_size,
|
||||
page_size=pool.slots_per_page,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"""Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook."""
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_dsa_index_kpool,
|
||||
is_deepseek_dsa,
|
||||
unwrap_modelopt_quantization_config,
|
||||
)
|
||||
@@ -139,8 +140,16 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"needs Triton>=3.5.0 or AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1)."
|
||||
)
|
||||
else:
|
||||
overrides["page_size"] = 64
|
||||
logger.warning("Setting page size to 64 for DeepSeek DSA.")
|
||||
index_kpool = get_dsa_index_kpool(hf_config)
|
||||
if index_kpool > 1 and getattr(cfg, "dcp_size", 1) > 1:
|
||||
raise ValueError(
|
||||
"--dcp-size > 1 is not supported for DSA with "
|
||||
f"index_kpool={index_kpool}: a pooled index key spans "
|
||||
"positions owned by different DCP ranks."
|
||||
)
|
||||
page_size = 64 * index_kpool
|
||||
overrides["page_size"] = page_size
|
||||
logger.warning(f"Setting page size to {page_size} for DeepSeek DSA.")
|
||||
elif get_platform().is_xpu:
|
||||
overrides["page_size"] = 128
|
||||
logger.warning("Setting page size to 128 for DeepSeek DSA on XPU.")
|
||||
|
||||
@@ -68,7 +68,7 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return dsa_impl
|
||||
|
||||
def _kpool_slots_per_page(self) -> int:
|
||||
return getattr(self.token_to_kv_pool, "slots_per_page", self.real_page_size)
|
||||
return self.token_to_kv_pool.slots_per_page
|
||||
|
||||
def _build_kpool_paged_mqa_schedule_metadata(self) -> bool:
|
||||
if self.device_sm_major == 9:
|
||||
|
||||
@@ -523,7 +523,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
)
|
||||
|
||||
pool_start_id = first_pos // kpool
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = attn_metadata.page_size
|
||||
use_returned_compressed = return_compressed and (
|
||||
not write_cache or pool_start_id == 0
|
||||
)
|
||||
@@ -955,7 +955,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
|
||||
|
||||
pool = get_token_to_kv_pool()
|
||||
page_size = pool.page_size
|
||||
page_size = pool.slots_per_page
|
||||
# DeepGEMM paged-MQA requires 64-token pages.
|
||||
assert page_size == 64, "only support page size 64"
|
||||
|
||||
@@ -1241,7 +1241,8 @@ class IndexerKPool(MultiPlatformOp):
|
||||
)
|
||||
|
||||
pool_size = self.index_kpool
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = metadata.attn_metadata.page_size
|
||||
slots_per_page = get_token_to_kv_pool().slots_per_page
|
||||
token_nums = q_fp8.shape[0]
|
||||
tail_pool = pool_size - 1
|
||||
topk_result = torch.empty(
|
||||
@@ -1355,7 +1356,9 @@ class IndexerKPool(MultiPlatformOp):
|
||||
token_page_table = block_tables[
|
||||
i, :num_token_pages
|
||||
].contiguous()
|
||||
pool_pages = (curr_pool_start + page_size - 1) // page_size
|
||||
pool_pages = (
|
||||
curr_pool_start + slots_per_page - 1
|
||||
) // slots_per_page
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
token_page_table, pool_size
|
||||
)[:pool_pages].contiguous()
|
||||
@@ -1395,7 +1398,9 @@ class IndexerKPool(MultiPlatformOp):
|
||||
token_page_table = block_tables[
|
||||
i, :num_token_pages
|
||||
].contiguous()
|
||||
pool_pages = (pool_seq_len + page_size - 1) // page_size
|
||||
pool_pages = (
|
||||
pool_seq_len + slots_per_page - 1
|
||||
) // slots_per_page
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
token_page_table, pool_size
|
||||
)[:pool_pages].contiguous()
|
||||
@@ -1494,7 +1499,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
|
||||
assert forward_batch.forward_mode.is_extend_without_speculative()
|
||||
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = get_token_to_kv_pool().slots_per_page
|
||||
assert page_size == 64, "only support page size 64"
|
||||
assert len(weights.shape) == 3
|
||||
weights = weights.squeeze(-1)
|
||||
|
||||
@@ -64,7 +64,7 @@ def build_pooled_page_table_64(
|
||||
idx = torch.arange(
|
||||
0, page_table_64.shape[-1], pool_size, device=page_table_64.device
|
||||
)
|
||||
return page_table_64[..., idx]
|
||||
return page_table_64[..., idx] // pool_size
|
||||
|
||||
|
||||
def gather_index_k_scale_prefix_into(
|
||||
@@ -79,7 +79,7 @@ def gather_index_k_scale_prefix_into(
|
||||
assert page_indices.dtype in (torch.int32, torch.int64)
|
||||
assert k_out.dtype == torch.uint8
|
||||
assert scale_out.dtype == torch.float32
|
||||
assert pool.page_size == BLOCK_SIZE_K
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert k_out.shape[0] >= seq_len
|
||||
assert k_out.shape[1] == INDEX_HEAD_DIM
|
||||
assert scale_out.shape[0] >= seq_len
|
||||
@@ -96,11 +96,11 @@ def gather_index_k_scale_prefix_into(
|
||||
page_indices,
|
||||
k_out,
|
||||
scale_out,
|
||||
PAGE_SIZE=pool.page_size,
|
||||
PAGE_SIZE=pool.slots_per_page,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * INDEX_HEAD_DIM,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
)
|
||||
|
||||
@@ -159,8 +159,6 @@ def kpool_build_ragged_layout(
|
||||
total_q: int,
|
||||
pool_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# One packed-cache page represents 64 pools, so its source is every
|
||||
# pool_size-th real token page.
|
||||
device = full_page_table.device
|
||||
n_rag = cu_pages_excl.shape[0]
|
||||
concat_page_table = torch.empty(
|
||||
@@ -224,7 +222,11 @@ def _kpool_build_ragged_layout_kernel(
|
||||
mask=p_mask,
|
||||
other=0,
|
||||
)
|
||||
tl.store(concat_page_table_ptr + page_start + p_offs, pages, mask=p_mask)
|
||||
tl.store(
|
||||
concat_page_table_ptr + page_start + p_offs,
|
||||
pages // POOL_SIZE,
|
||||
mask=p_mask,
|
||||
)
|
||||
|
||||
for q_off in tl.range(0, BLOCK_Q * tl.cdiv(q_count, BLOCK_Q), BLOCK_Q):
|
||||
q_offs = q_off + tl.arange(0, BLOCK_Q)
|
||||
@@ -377,11 +379,14 @@ def _update_kpool_write_plan_kernel(
|
||||
)
|
||||
# Promote the row term before multiplication; 1M-context strides
|
||||
# overflow int32 around row 2048.
|
||||
packed_page = tl.load(
|
||||
real_page_table_ptr
|
||||
+ (b * N).to(tl.int64) * real_page_table_stride_0
|
||||
+ token_page_row
|
||||
).to(tl.int64)
|
||||
packed_page = (
|
||||
tl.load(
|
||||
real_page_table_ptr
|
||||
+ (b * N).to(tl.int64) * real_page_table_stride_0
|
||||
+ token_page_row
|
||||
).to(tl.int64)
|
||||
// POOL_SIZE
|
||||
)
|
||||
write_loc = packed_page * SLOTS_PER_PAGE + (pool_id % SLOTS_PER_PAGE)
|
||||
tl.store(
|
||||
write_loc_out_ptr + b * write_loc_out_stride_0 + p,
|
||||
@@ -398,10 +403,11 @@ def compute_pooled_write_locs(
|
||||
pool_ids = pool_ids.to(torch.int64)
|
||||
pool_page_group = torch.div(pool_ids, BLOCK_SIZE_K, rounding_mode="floor")
|
||||
token_page_row = pool_page_group * pool_size
|
||||
packed_page = page_table_64.index_select(0, token_page_row.to(torch.int64))
|
||||
return packed_page.to(torch.int64) * BLOCK_SIZE_K + torch.remainder(
|
||||
pool_ids, BLOCK_SIZE_K
|
||||
packed_page = (
|
||||
page_table_64.index_select(0, token_page_row.to(torch.int64)).to(torch.int64)
|
||||
// pool_size
|
||||
)
|
||||
return packed_page * BLOCK_SIZE_K + torch.remainder(pool_ids, BLOCK_SIZE_K)
|
||||
|
||||
|
||||
def history_group_budget_for_topk(topk: int, pool_size: int) -> int:
|
||||
@@ -712,7 +718,7 @@ def kpool_softmax_rotate_write_cache(
|
||||
assert slot_score.dtype in KPOOL_SCORE_DTYPES
|
||||
assert ape.dtype == torch.float32
|
||||
assert buf.dtype == torch.uint8
|
||||
assert pool.page_size == BLOCK_SIZE_K
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert pool.index_head_dim == INDEX_HEAD_DIM
|
||||
assert loc.dtype == torch.int64
|
||||
assert write_cache or return_compressed
|
||||
@@ -771,12 +777,12 @@ def kpool_softmax_rotate_write_cache(
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape.stride(0),
|
||||
PAGE_SIZE=pool.page_size,
|
||||
PAGE_SIZE=pool.slots_per_page,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=slot_k.shape[2],
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
@@ -816,7 +822,7 @@ def kpool_decode_update_and_maybe_write_cache(
|
||||
assert slot_score.dtype == tail_score.dtype
|
||||
assert ape.dtype == torch.float32
|
||||
assert buf.dtype == torch.uint8
|
||||
assert pool.page_size == BLOCK_SIZE_K
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert pool.index_head_dim == INDEX_HEAD_DIM
|
||||
assert tail_k.is_contiguous()
|
||||
assert tail_score.is_contiguous()
|
||||
@@ -1165,7 +1171,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
+ row * block_tables_stride_0
|
||||
+ token_page_row * block_tables_stride_1,
|
||||
)
|
||||
loc_page_index = packed_page.to(tl.int64)
|
||||
loc_page_index = packed_page.to(tl.int64) // POOL_SIZE
|
||||
loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
|
||||
@@ -373,8 +373,9 @@ def _kpool_plan_to_gpu(
|
||||
pool_pool_id_t, slots_per_page, rounding_mode="floor"
|
||||
)
|
||||
token_page_row = pool_page_group * pool_size
|
||||
packed_page = full_real_page_table[pool_batch_idx_t, token_page_row].to(
|
||||
torch.int64
|
||||
packed_page = (
|
||||
full_real_page_table[pool_batch_idx_t, token_page_row].to(torch.int64)
|
||||
// pool_size
|
||||
)
|
||||
pool_write_locs = packed_page * slots_per_page + torch.remainder(
|
||||
pool_pool_id_t, slots_per_page
|
||||
|
||||
@@ -369,7 +369,6 @@ class DeepseekSparseAttnBackend(
|
||||
self.forward_metadata: DSAMetadata
|
||||
self.device = model_runner.device
|
||||
assert isinstance(model_runner.page_size, int)
|
||||
self.real_page_size = model_runner.page_size
|
||||
self.num_splits = (
|
||||
1 if get_exec().deterministic.enable_deterministic_inference else 0
|
||||
)
|
||||
@@ -381,6 +380,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.real_page_size = model_runner.page_size // self.dsa_index_kpool
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self._memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
|
||||
@@ -35,7 +35,7 @@ class IndexKeyCache:
|
||||
pool = self.pool
|
||||
return (
|
||||
num_pages,
|
||||
pool.page_size
|
||||
pool.slots_per_page
|
||||
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
|
||||
)
|
||||
|
||||
|
||||
@@ -5146,7 +5146,10 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
self.index_kpool = index_kpool
|
||||
self.index_kpool_compress = index_kpool_compress
|
||||
self.tail_extra_slots = tail_extra_slots
|
||||
self.slots_per_page = self.page_size
|
||||
assert self.page_size % index_kpool == 0, (
|
||||
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
|
||||
)
|
||||
self.slots_per_page = self.page_size // index_kpool
|
||||
if index_buf_size is None:
|
||||
index_buf_size = size
|
||||
self.index_buf_size = index_buf_size
|
||||
@@ -5160,22 +5163,26 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
)
|
||||
assert len(self.skip_topk_layers) == layer_num
|
||||
|
||||
physical_page_size = self.page_size // index_kpool
|
||||
if _is_hip:
|
||||
if aiter_can_use_preshuffle_paged_mqa():
|
||||
assert self.page_size % 16 == 0, (
|
||||
f"HIP preshuffle requires page_size to be a multiple of 16, got {self.page_size}"
|
||||
assert physical_page_size % 16 == 0, (
|
||||
f"HIP preshuffle requires page_size to be a multiple of 16, got {physical_page_size}"
|
||||
)
|
||||
else:
|
||||
assert self.page_size == 1, (
|
||||
f"HIP legacy DSA path requires page_size == 1, got {self.page_size}"
|
||||
assert physical_page_size == 1, (
|
||||
f"HIP legacy DSA path requires page_size == 1, got {physical_page_size}"
|
||||
)
|
||||
elif is_xpu():
|
||||
assert self.page_size in (
|
||||
assert physical_page_size in (
|
||||
64,
|
||||
128,
|
||||
), f"XPU DSA requires page_size 64 or 128, got {self.page_size}"
|
||||
), f"XPU DSA requires page_size 64 or 128, got {physical_page_size}"
|
||||
else:
|
||||
assert self.page_size == 64
|
||||
assert physical_page_size == 64, (
|
||||
f"DSA requires 64-token physical pages, got page_size={self.page_size} "
|
||||
f"with index_kpool={index_kpool}"
|
||||
)
|
||||
self.index_key_cache = self._create_index_key_cache()
|
||||
self._init_kpool_compress_tail_buffers(
|
||||
index_kpool=index_kpool,
|
||||
|
||||
@@ -98,16 +98,21 @@ def make_dsa_indexer_pool_decl(
|
||||
pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER
|
||||
) -> HostPoolDecl:
|
||||
"""Index key buffers riding on the full-KV pages: indices and layout both follow KV."""
|
||||
index_page_bytes = pool.slots_per_page * dsa_indexer_bytes_per_token_per_layer(
|
||||
index_head_dim=pool.index_head_dim,
|
||||
quant_block_size=pool.quant_block_size,
|
||||
)
|
||||
assert index_page_bytes % pool.page_size == 0, (
|
||||
f"index page of {index_page_bytes} bytes does not split over "
|
||||
f"page_size={pool.page_size} tokens"
|
||||
)
|
||||
return HostPoolDecl(
|
||||
pool_name=name,
|
||||
device_pool=pool,
|
||||
indices_from_pool=PoolName.KV,
|
||||
layout_source=PoolName.KV,
|
||||
storage_info=HostPoolStorageInfo(
|
||||
bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer(
|
||||
index_head_dim=pool.index_head_dim,
|
||||
quant_block_size=pool.quant_block_size,
|
||||
),
|
||||
bytes_per_token_per_layer=index_page_bytes // pool.page_size,
|
||||
dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype,
|
||||
),
|
||||
host_pool_builder=DSAIndexerHostPoolBuilder(),
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.configs.model_config import (
|
||||
AttentionArch,
|
||||
dsa_layer_skips_topk,
|
||||
get_dsa_index_head_dim,
|
||||
get_dsa_index_kpool,
|
||||
get_minimax_sparse_attention_config,
|
||||
get_minimax_sparse_disable_value_layer_ids,
|
||||
get_minimax_sparse_layer_ids,
|
||||
@@ -515,8 +516,9 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
allocate_all_layers: bool = False,
|
||||
) -> int:
|
||||
index_head_dim = get_dsa_index_head_dim(kvc.model_config.hf_config)
|
||||
indexer_size_per_token = (
|
||||
index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4
|
||||
indexer_size_per_token = ceil_div(
|
||||
index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4,
|
||||
get_dsa_index_kpool(kvc.model_config.hf_config),
|
||||
)
|
||||
element_size = torch._utils._element_size(
|
||||
DSATokenToKVPool.index_k_with_scale_buffer_dtype
|
||||
@@ -642,9 +644,10 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
if is_deepseek_dsa(model_config.hf_config):
|
||||
index_head_dim = get_dsa_index_head_dim(model_config.hf_config)
|
||||
index_elements = (
|
||||
index_elements = ceil_div(
|
||||
index_head_dim
|
||||
+ index_head_dim // DSATokenToKVPool.quant_block_size * 4
|
||||
+ index_head_dim // DSATokenToKVPool.quant_block_size * 4,
|
||||
get_dsa_index_kpool(model_config.hf_config),
|
||||
)
|
||||
self._full_per_token += index_elements * torch._utils._element_size(
|
||||
DSATokenToKVPool.index_k_with_scale_buffer_dtype
|
||||
|
||||
@@ -24,7 +24,7 @@ register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-l
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "Test requires CUDA")
|
||||
class TestDsaKpoolMultiPool(CustomTestCase):
|
||||
POOL_SIZE = 4
|
||||
PAGE_SIZE = 64
|
||||
PAGE_SIZE = 256
|
||||
SLOTS_PER_PAGE = 64
|
||||
NUM_DRAFT_TOKENS = 6
|
||||
|
||||
@@ -64,10 +64,10 @@ class TestDsaKpoolMultiPool(CustomTestCase):
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
real_page_table[:num_draft_tokens, 0] = 2
|
||||
real_page_table[:num_draft_tokens, 4] = 3
|
||||
real_page_table[num_draft_tokens:, 0] = 5
|
||||
real_page_table[num_draft_tokens:, 4] = 6
|
||||
real_page_table[:num_draft_tokens, 0] = 2 * self.POOL_SIZE
|
||||
real_page_table[:num_draft_tokens, 4] = 3 * self.POOL_SIZE
|
||||
real_page_table[num_draft_tokens:, 0] = 5 * self.POOL_SIZE
|
||||
real_page_table[num_draft_tokens:, 4] = 6 * self.POOL_SIZE
|
||||
|
||||
update_kpool_write_plan_cuda_graph(
|
||||
write_start=write_start,
|
||||
|
||||
@@ -24,7 +24,7 @@ class TestKpoolPreshuffledCacheLayout(CustomTestCase):
|
||||
"""A layout mismatch between the two writers never raises; it silently
|
||||
returns the wrong top-k."""
|
||||
torch.manual_seed(0)
|
||||
pool = SimpleNamespace(page_size=PAGE_SIZE)
|
||||
pool = SimpleNamespace(page_size=4 * PAGE_SIZE, slots_per_page=PAGE_SIZE)
|
||||
num_pages, seq_len = 2, PAGE_SIZE + 5
|
||||
buf = torch.zeros(
|
||||
num_pages,
|
||||
|
||||
@@ -671,6 +671,7 @@ class TestKPoolPerRequestChunking(CustomTestCase):
|
||||
get_seqlens_expanded=lambda: seqlens_expanded,
|
||||
topk_transform_method=TopkTransformMethod.RAGGED,
|
||||
attn_metadata=SimpleNamespace(
|
||||
page_size=64,
|
||||
topk_indices_offset=torch.arange(token_nums, dtype=torch.int32) + 100,
|
||||
),
|
||||
)
|
||||
@@ -689,7 +690,8 @@ class TestKPoolPerRequestChunking(CustomTestCase):
|
||||
index_kpool=self.POOL,
|
||||
)
|
||||
with _chunking_patches(
|
||||
rows_per_chunk=rows_per_chunk, kv_pool=SimpleNamespace(page_size=64)
|
||||
rows_per_chunk=rows_per_chunk,
|
||||
kv_pool=SimpleNamespace(page_size=256, slots_per_page=64),
|
||||
):
|
||||
result = IndexerKPool._get_topk_ragged_kpool(
|
||||
backend,
|
||||
|
||||
@@ -268,6 +268,8 @@ class TestDraftSidecarPoolDispatch(CustomTestCase):
|
||||
draft_kv_pool.layer_num = 1
|
||||
draft_kv_pool.size = 800
|
||||
draft_kv_pool.index_head_dim = 128
|
||||
draft_kv_pool.page_size = 64
|
||||
draft_kv_pool.slots_per_page = 64
|
||||
draft_kv_pool.skip_topk_layers = [False]
|
||||
draft_kv_pool.index_key_cache = SimpleNamespace(buffer=[object()])
|
||||
draft_host_pool = SimpleNamespace(layer_num=1)
|
||||
@@ -333,6 +335,8 @@ def _dsa_pool_stub(*, layer_num: int, size: int = 4096, shard: tuple | None = No
|
||||
pool.qk_rope_head_dim = 64
|
||||
pool.kv_cache_dim = 576
|
||||
pool.index_head_dim = 128
|
||||
pool.page_size = 64
|
||||
pool.slots_per_page = 64
|
||||
pool.skip_topk_layers = [False] * layer_num
|
||||
pool.index_key_cache = SimpleNamespace(buffer=[object()] * layer_num)
|
||||
pool.layer_shard_enabled = shard is not None
|
||||
|
||||
@@ -483,6 +483,8 @@ class TestDSAIndexerPoolDecl(CustomTestCase):
|
||||
qk_rope_head_dim=64,
|
||||
index_head_dim=128,
|
||||
quant_block_size=128,
|
||||
page_size=64,
|
||||
slots_per_page=64,
|
||||
skip_topk_layers=[False] * 5,
|
||||
)
|
||||
|
||||
|
||||
@@ -83,6 +83,8 @@ class TestMLAHostDedupPrimitives(unittest.TestCase):
|
||||
end_layer=2,
|
||||
index_head_dim=8,
|
||||
quant_block_size=4,
|
||||
page_size=2,
|
||||
slots_per_page=2,
|
||||
skip_topk_layers=[False] * 2,
|
||||
)
|
||||
indexer_host = DSAIndexerPoolHost(
|
||||
|
||||
Reference in New Issue
Block a user