[DSA] Clarify pooled-page naming and ownership (#42671)
This commit is contained in:
@@ -41,7 +41,7 @@ class GetK:
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
page_size = pool.slots_per_page
|
||||
page_size = pool.index_page_size
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
index_k_fp8 = torch.empty(
|
||||
@@ -71,7 +71,7 @@ class GetK:
|
||||
# page_indices: (num_pages,), element := a page index
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
num_k_bytes_per_page = pool.slots_per_page * pool.index_head_dim
|
||||
num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim
|
||||
num_k_bytes_per_token = pool.index_head_dim
|
||||
|
||||
# buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8
|
||||
@@ -100,7 +100,7 @@ class GetK:
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.slots_per_page,
|
||||
page_size=pool.index_page_size,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -114,7 +114,7 @@ class GetS:
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
page_size = pool.slots_per_page
|
||||
page_size = pool.index_page_size
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
assert pool.index_head_dim // pool.quant_block_size == 1
|
||||
@@ -140,7 +140,7 @@ class GetS:
|
||||
"""
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
s_offset_in_page = pool.slots_per_page * pool.index_head_dim
|
||||
s_offset_in_page = pool.index_page_size * pool.index_head_dim
|
||||
num_s_bytes_per_page = buf.shape[1] - s_offset_in_page
|
||||
num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4
|
||||
|
||||
@@ -170,7 +170,7 @@ class GetS:
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.slots_per_page,
|
||||
page_size=pool.index_page_size,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -198,7 +198,7 @@ class GetKAndS:
|
||||
):
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
|
||||
|
||||
page_size = pool.slots_per_page
|
||||
page_size = pool.index_page_size
|
||||
index_head_dim = pool.index_head_dim
|
||||
quant_block_size = pool.quant_block_size
|
||||
scale_elems = index_head_dim // quant_block_size
|
||||
@@ -254,7 +254,7 @@ class GetKAndS:
|
||||
seq_lens=seq_len_tensor,
|
||||
seq_len_sum=seq_len_sum,
|
||||
max_seq_len=max_seq_len,
|
||||
page_size=pool.slots_per_page,
|
||||
page_size=pool.index_page_size,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
@@ -273,7 +273,7 @@ class SetKAndS:
|
||||
loc=loc,
|
||||
index_k=index_k,
|
||||
index_k_scale=index_k_scale,
|
||||
page_size=pool.slots_per_page,
|
||||
page_size=pool.index_page_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ def _jit_kpool_topk_transform_module(group_topk: int) -> Module:
|
||||
def fast_kpool_topk_transform_fused(
|
||||
score: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
topk: int,
|
||||
page_table: Optional[torch.Tensor] = None,
|
||||
topk_indices_offset: Optional[torch.Tensor] = None,
|
||||
@@ -37,8 +37,8 @@ def fast_kpool_topk_transform_fused(
|
||||
seq_lens: Optional[torch.Tensor] = None,
|
||||
page_table_row_index: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert topk % pool_size == 0
|
||||
group_topk = topk // pool_size
|
||||
assert topk % kpool == 0
|
||||
group_topk = topk // kpool
|
||||
assert group_topk in SUPPORTED_GROUP_TOPK, (
|
||||
"fast_kpool_topk_transform supports pool-level topk "
|
||||
f"{SUPPORTED_GROUP_TOPK}, got {group_topk}"
|
||||
@@ -53,7 +53,7 @@ def fast_kpool_topk_transform_fused(
|
||||
assert page_table_row_index.dim() == 1
|
||||
assert page_table_row_index.shape[0] == score.shape[0]
|
||||
|
||||
out_cols = topk + (pool_size - 1 if seq_lens is not None else 0)
|
||||
out_cols = topk + (kpool - 1 if seq_lens is not None else 0)
|
||||
dst_token_indices = score.new_empty((score.shape[0], out_cols), dtype=torch.int32)
|
||||
|
||||
module = _jit_kpool_topk_transform_module(group_topk)
|
||||
@@ -61,7 +61,7 @@ def fast_kpool_topk_transform_fused(
|
||||
score,
|
||||
lengths,
|
||||
dst_token_indices,
|
||||
pool_size,
|
||||
kpool,
|
||||
page_table,
|
||||
topk_indices_offset,
|
||||
row_starts,
|
||||
|
||||
@@ -40,7 +40,7 @@ def _fused_dsa_decode_metadata_kernel(
|
||||
num_splits,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
index_kpool: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGE_TABLE_1: tl.constexpr,
|
||||
BLOCK_BS: tl.constexpr,
|
||||
@@ -111,13 +111,13 @@ def _fused_dsa_decode_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ row_i64 * real_page_table_stride_0
|
||||
+ real_cols * real_page_table_stride_1,
|
||||
vals // real_page_size,
|
||||
vals // physical_page_size,
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -135,16 +135,16 @@ def fused_dsa_decode_metadata(
|
||||
bs: int,
|
||||
max_len: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
index_kpool: int = 1,
|
||||
) -> None:
|
||||
"""Fill decode-graph DSA metadata (seqlens + page tables) from req_to_token.
|
||||
|
||||
``page_table_1`` (the wide page_size=1 table) is optional: pass ``None`` to
|
||||
skip materializing it and write only the compact ``real_page_table``
|
||||
(page_size=``real_page_size``). This is used by the fused decode CUDA graph,
|
||||
(page_size=``physical_page_size``). This is used by the fused decode CUDA graph,
|
||||
where the wide table is never read (attention uses topk_indices, the indexer
|
||||
uses real_page_table); ``real_page_size`` must be >1 in that case. When a
|
||||
uses real_page_table); ``physical_page_size`` must be >1 in that case. When a
|
||||
tensor is passed, behavior is unchanged (both tables are written).
|
||||
|
||||
Contract: each page-table row is written only over its live prefix
|
||||
@@ -170,7 +170,7 @@ def fused_dsa_decode_metadata(
|
||||
return
|
||||
assert index_kpool > 0
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -217,7 +217,7 @@ def fused_dsa_decode_metadata(
|
||||
num_splits,
|
||||
dsa_index_topk,
|
||||
index_kpool,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
has_real_page_table,
|
||||
has_page_table_1,
|
||||
BLOCK_BS=block_bs,
|
||||
|
||||
@@ -45,7 +45,7 @@ def _fused_dsa_draft_extend_metadata_kernel(
|
||||
num_splits,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
index_kpool: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGE_TABLE_1: tl.constexpr,
|
||||
STATIC_EXTEND_LEN: tl.constexpr,
|
||||
@@ -183,13 +183,13 @@ def _fused_dsa_draft_extend_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n[None, :] % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n[None, :] % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ out_rows_i64[:, None] * real_page_table_stride_0
|
||||
+ real_cols[None, :] * real_page_table_stride_1,
|
||||
(vals // real_page_size)[None, :],
|
||||
(vals // physical_page_size)[None, :],
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -210,7 +210,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
total_len: int,
|
||||
max_seqlen_k: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
max_extend_len: int,
|
||||
max_total_len: int,
|
||||
static_extend_len: bool = False,
|
||||
@@ -245,7 +245,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
assert total_len <= bs * max_extend_len
|
||||
assert index_kpool > 0
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -297,7 +297,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
num_splits,
|
||||
dsa_index_topk,
|
||||
index_kpool,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
has_real_page_table,
|
||||
has_page_table_1,
|
||||
static_extend_len,
|
||||
|
||||
@@ -44,7 +44,7 @@ def _fused_dsa_target_verify_metadata_kernel(
|
||||
num_splits,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
index_kpool: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
next_n: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGED_MQA_CTX_LENS: tl.constexpr,
|
||||
@@ -146,13 +146,13 @@ def _fused_dsa_target_verify_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ out_row_i64 * real_page_table_stride_0
|
||||
+ real_cols * real_page_table_stride_1,
|
||||
vals // real_page_size,
|
||||
vals // physical_page_size,
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -171,7 +171,7 @@ def _prep_fused_dsa_target_verify_metadata_launch(
|
||||
bs: int,
|
||||
max_seqlen_k: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
next_n: int,
|
||||
paged_mqa_ctx_lens_2d: torch.Tensor = None,
|
||||
index_kpool: int = 1,
|
||||
@@ -189,7 +189,7 @@ def _prep_fused_dsa_target_verify_metadata_launch(
|
||||
assert next_n > 0
|
||||
assert index_kpool > 0
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -251,7 +251,7 @@ def _prep_fused_dsa_target_verify_metadata_launch(
|
||||
num_splits,
|
||||
dsa_index_topk,
|
||||
index_kpool,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
next_n,
|
||||
has_real_page_table,
|
||||
has_paged_mqa_ctx_lens,
|
||||
@@ -279,7 +279,7 @@ def fused_dsa_target_verify_metadata(
|
||||
bs: int,
|
||||
max_seqlen_k: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
next_n: int,
|
||||
paged_mqa_ctx_lens_2d: torch.Tensor = None,
|
||||
index_kpool: int = 1,
|
||||
@@ -305,7 +305,7 @@ def fused_dsa_target_verify_metadata(
|
||||
bs,
|
||||
max_seqlen_k,
|
||||
dsa_index_topk,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
next_n,
|
||||
paged_mqa_ctx_lens_2d,
|
||||
index_kpool,
|
||||
|
||||
@@ -33,7 +33,7 @@ def _fused_dsa_decode_metadata_kernel(
|
||||
bs: tl.constexpr,
|
||||
max_len,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGE_TABLE_1: tl.constexpr,
|
||||
BLOCK_BS: tl.constexpr,
|
||||
@@ -98,13 +98,13 @@ def _fused_dsa_decode_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ row.to(tl.int64) * real_page_table_stride_0
|
||||
+ real_cols * real_page_table_stride_1,
|
||||
vals // real_page_size,
|
||||
vals // physical_page_size,
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -122,15 +122,15 @@ def fused_dsa_decode_metadata(
|
||||
bs: int,
|
||||
max_len: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
) -> None:
|
||||
"""Fill decode-graph DSA metadata (seqlens + page tables) from req_to_token.
|
||||
|
||||
``page_table_1`` (the wide page_size=1 table) is optional: pass ``None`` to
|
||||
skip materializing it and write only the compact ``real_page_table``
|
||||
(page_size=``real_page_size``). This is used by the fused decode CUDA graph,
|
||||
(page_size=``physical_page_size``). This is used by the fused decode CUDA graph,
|
||||
where the wide table is never read (attention uses topk_indices, the indexer
|
||||
uses real_page_table); ``real_page_size`` must be >1 in that case. When a
|
||||
uses real_page_table); ``physical_page_size`` must be >1 in that case. When a
|
||||
tensor is passed, behavior is unchanged (both tables are written).
|
||||
|
||||
Contract: each page-table row is written only over its live prefix
|
||||
@@ -150,7 +150,7 @@ def fused_dsa_decode_metadata(
|
||||
dsa_cu_seqlens_k[:1].zero_()
|
||||
return
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -194,7 +194,7 @@ def fused_dsa_decode_metadata(
|
||||
bs,
|
||||
max_len,
|
||||
dsa_index_topk,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
has_real_page_table,
|
||||
has_page_table_1,
|
||||
BLOCK_BS=block_bs,
|
||||
@@ -234,7 +234,7 @@ def _fused_dsa_target_verify_metadata_kernel(
|
||||
bs: tl.constexpr,
|
||||
max_seqlen_k,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
next_n: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGED_MQA_CTX_LENS: tl.constexpr,
|
||||
@@ -330,13 +330,13 @@ def _fused_dsa_target_verify_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ out_row.to(tl.int64) * real_page_table_stride_0
|
||||
+ real_cols * real_page_table_stride_1,
|
||||
vals // real_page_size,
|
||||
vals // physical_page_size,
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -355,7 +355,7 @@ def fused_dsa_target_verify_metadata(
|
||||
bs: int,
|
||||
max_seqlen_k: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
next_n: int,
|
||||
paged_mqa_ctx_lens_2d: torch.Tensor = None,
|
||||
) -> None:
|
||||
@@ -374,7 +374,7 @@ def fused_dsa_target_verify_metadata(
|
||||
return
|
||||
assert next_n > 0
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -433,7 +433,7 @@ def fused_dsa_target_verify_metadata(
|
||||
bs,
|
||||
max_seqlen_k,
|
||||
dsa_index_topk,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
next_n,
|
||||
has_real_page_table,
|
||||
has_paged_mqa_ctx_lens,
|
||||
@@ -477,7 +477,7 @@ def _fused_dsa_draft_extend_metadata_kernel(
|
||||
total_len,
|
||||
max_seqlen_k,
|
||||
dsa_index_topk: tl.constexpr,
|
||||
real_page_size: tl.constexpr,
|
||||
physical_page_size: tl.constexpr,
|
||||
HAS_REAL_PAGE_TABLE: tl.constexpr,
|
||||
HAS_PAGE_TABLE_1: tl.constexpr,
|
||||
STATIC_EXTEND_LEN: tl.constexpr,
|
||||
@@ -603,13 +603,13 @@ def _fused_dsa_draft_extend_metadata_kernel(
|
||||
)
|
||||
|
||||
if HAS_REAL_PAGE_TABLE:
|
||||
real_mask = mask & ((offs_n[None, :] % real_page_size) == 0)
|
||||
real_cols = offs_n // real_page_size
|
||||
real_mask = mask & ((offs_n[None, :] % physical_page_size) == 0)
|
||||
real_cols = offs_n // physical_page_size
|
||||
tl.store(
|
||||
real_page_table
|
||||
+ out_rows.to(tl.int64)[:, None] * real_page_table_stride_0
|
||||
+ real_cols[None, :] * real_page_table_stride_1,
|
||||
(vals // real_page_size)[None, :],
|
||||
(vals // physical_page_size)[None, :],
|
||||
mask=real_mask,
|
||||
)
|
||||
|
||||
@@ -630,7 +630,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
total_len: int,
|
||||
max_seqlen_k: int,
|
||||
dsa_index_topk: int,
|
||||
real_page_size: int,
|
||||
physical_page_size: int,
|
||||
max_extend_len: int,
|
||||
max_total_len: int,
|
||||
static_extend_len: bool = False,
|
||||
@@ -663,7 +663,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
assert max_extend_len > 0
|
||||
assert total_len <= bs * max_extend_len
|
||||
|
||||
has_real_page_table = real_page_size > 1
|
||||
has_real_page_table = physical_page_size > 1
|
||||
if has_real_page_table:
|
||||
assert real_page_table is not None
|
||||
assert real_page_table.is_cuda
|
||||
@@ -712,7 +712,7 @@ def fused_dsa_draft_extend_metadata(
|
||||
total_len,
|
||||
max_seqlen_k,
|
||||
dsa_index_topk,
|
||||
real_page_size,
|
||||
physical_page_size,
|
||||
has_real_page_table,
|
||||
has_page_table_1,
|
||||
static_extend_len,
|
||||
|
||||
@@ -141,12 +141,14 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
else:
|
||||
index_kpool = get_dsa_index_kpool(hf_config)
|
||||
if index_kpool > 1 and getattr(cfg, "dcp_size", 1) > 1:
|
||||
if index_kpool > 1 and cfg.dcp_size > 1:
|
||||
raise ValueError(
|
||||
"--dcp-size > 1 is not supported for DSA with "
|
||||
f"index_kpool={index_kpool}: a pooled index key spans "
|
||||
"positions owned by different DCP ranks."
|
||||
)
|
||||
# A logical page must own a whole pooled index group so that a
|
||||
# prefix hit never ends inside one.
|
||||
page_size = 64 * index_kpool
|
||||
overrides["page_size"] = page_size
|
||||
logger.warning(f"Setting page size to {page_size} for DeepSeek DSA.")
|
||||
|
||||
@@ -67,8 +67,8 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return "fa3"
|
||||
return dsa_impl
|
||||
|
||||
def _kpool_slots_per_page(self) -> int:
|
||||
return self.token_to_kv_pool.slots_per_page
|
||||
def _kpool_index_page_size(self) -> int:
|
||||
return self.token_to_kv_pool.index_page_size
|
||||
|
||||
def _build_kpool_paged_mqa_schedule_metadata(self) -> bool:
|
||||
if self.device_sm_major == 9:
|
||||
@@ -86,7 +86,7 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return metadata
|
||||
|
||||
forward_mode = forward_batch.forward_mode
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
index_page_size = self._kpool_index_page_size()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
if forward_mode.is_extend_without_speculative():
|
||||
assert topk_transform_method is not None
|
||||
@@ -94,9 +94,9 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return init_kpool_extend_metadata(
|
||||
metadata,
|
||||
forward_batch,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
index_page_size=index_page_size,
|
||||
topk_transform_method=topk_transform_method,
|
||||
full_real_page_table=kpool_inputs.full_real_page_table,
|
||||
full_seqlens_expanded=kpool_inputs.full_seqlens_expanded,
|
||||
@@ -106,9 +106,9 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -116,12 +116,12 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return init_kpool_write_plan(
|
||||
metadata,
|
||||
forward_batch,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
real_page_table=metadata.real_page_table,
|
||||
num_draft_tokens=1,
|
||||
write_start=(forward_batch.seq_lens - 1).to(torch.int32),
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -129,12 +129,12 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return init_kpool_write_plan(
|
||||
metadata,
|
||||
forward_batch,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
real_page_table=metadata.real_page_table,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
write_start=forward_batch.seq_lens.to(torch.int32),
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -149,14 +149,14 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
return init_kpool_write_plan(
|
||||
metadata,
|
||||
forward_batch,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
real_page_table=metadata.real_page_table,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
write_start=(
|
||||
forward_batch.seq_lens - self.speculative_num_draft_tokens
|
||||
).to(torch.int32),
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
effective_n_per_batch=effective_n_per_batch,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
@@ -169,15 +169,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
if self.dsa_index_kpool <= 1:
|
||||
return metadata
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
index_page_size = self._kpool_index_page_size()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -191,14 +191,14 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
metadata = init_kpool_write_plan_capture(
|
||||
metadata,
|
||||
max_bs=bs,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
num_draft_tokens=(
|
||||
1 if is_decode else self.speculative_num_draft_tokens
|
||||
),
|
||||
device=self.device,
|
||||
is_verify=not is_decode,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
is_v2=is_v2,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
@@ -216,15 +216,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
if self.dsa_index_kpool <= 1:
|
||||
return
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
index_page_size = self._kpool_index_page_size()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -250,11 +250,11 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
write_start=write_start,
|
||||
req_pool_indices=req_pool_indices,
|
||||
real_page_table=metadata.real_page_table,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
num_draft_tokens=(1 if is_decode else self.speculative_num_draft_tokens),
|
||||
forward_mode=forward_mode,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
effective_n_per_batch=effective_n_per_batch,
|
||||
)
|
||||
|
||||
@@ -267,15 +267,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
if self.dsa_index_kpool <= 1:
|
||||
return
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
index_page_size = self._kpool_index_page_size()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
precomputed.cache_seqlens,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
index_page_size=index_page_size,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
@@ -294,9 +294,9 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
write_start=write_start,
|
||||
req_pool_indices=precomputed.req_pool_indices,
|
||||
real_page_table=metadata.real_page_table,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
kpool=self.dsa_index_kpool,
|
||||
physical_page_size=self.physical_page_size,
|
||||
num_draft_tokens=self.speculative_num_draft_tokens if is_verify else 1,
|
||||
forward_mode=forward_mode,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
|
||||
@@ -134,8 +134,10 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
dsa_cu_seqlens_k = torch.empty(
|
||||
bs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if self.real_page_size > 1:
|
||||
real_cols = (max_len + self.real_page_size - 1) // self.real_page_size
|
||||
if self.physical_page_size > 1:
|
||||
real_cols = (
|
||||
max_len + self.physical_page_size - 1
|
||||
) // self.physical_page_size
|
||||
real_page_table = torch.empty(
|
||||
(bs, real_cols), dtype=torch.int32, device=self.device
|
||||
)
|
||||
@@ -157,7 +159,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
bs=bs,
|
||||
max_len=max_len,
|
||||
dsa_index_topk=self.dsa_index_topk,
|
||||
real_page_size=self.real_page_size,
|
||||
physical_page_size=self.physical_page_size,
|
||||
)
|
||||
seqlens_expanded = cache_seqlens
|
||||
seqlens_expanded_size = bs
|
||||
@@ -204,7 +206,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
dsa_cu_seqlens_k = compute_cu_seqlens(dsa_cache_seqlens)
|
||||
|
||||
# Transform page table if needed
|
||||
if self.real_page_size > 1:
|
||||
if self.physical_page_size > 1:
|
||||
real_page_table = self._transform_table_1_to_real(page_indices)
|
||||
else:
|
||||
real_page_table = None # Will use page_indices directly
|
||||
@@ -264,10 +266,10 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
if self.real_page_size > 1:
|
||||
if self.physical_page_size > 1:
|
||||
real_cols = (
|
||||
max_seqlen_k + self.real_page_size - 1
|
||||
) // self.real_page_size
|
||||
max_seqlen_k + self.physical_page_size - 1
|
||||
) // self.physical_page_size
|
||||
real_page_table = torch.empty(
|
||||
(seqlens_expanded_size, real_cols),
|
||||
dtype=torch.int32,
|
||||
@@ -292,7 +294,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
bs=bs,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
dsa_index_topk=self.dsa_index_topk,
|
||||
real_page_size=self.real_page_size,
|
||||
physical_page_size=self.physical_page_size,
|
||||
next_n=self.speculative_num_draft_tokens,
|
||||
)
|
||||
|
||||
@@ -355,7 +357,7 @@ class DeepseekSparseAttnBackendMTPPrecomputeMixin:
|
||||
dsa_cu_seqlens_k = compute_cu_seqlens(dsa_cache_seqlens)
|
||||
|
||||
# Transform page table
|
||||
if self.real_page_size > 1:
|
||||
if self.physical_page_size > 1:
|
||||
real_page_table = self._transform_table_1_to_real(page_indices)
|
||||
else:
|
||||
real_page_table = None
|
||||
|
||||
@@ -421,7 +421,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
max_req = int(active_req.max().item())
|
||||
min_loc = int(active_write_loc.min().item())
|
||||
max_loc = int(active_write_loc.max().item())
|
||||
max_page = max_loc // int(pool.slots_per_page)
|
||||
max_page = max_loc // int(pool.index_page_size)
|
||||
if (
|
||||
min_chunk < 0
|
||||
or max_chunk >= key.shape[0]
|
||||
@@ -439,7 +439,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
f"{key.shape=}, {gate_score.shape=}, {tail_k_buf.shape=}, "
|
||||
f"{pool.get_index_k_with_scale_buffer(layer_id=layer_id).shape=}, "
|
||||
f"{min_chunk=}, {max_chunk=}, {min_req=}, {max_req=}, "
|
||||
f"{min_loc=}, {max_loc=}, {pool.slots_per_page=}"
|
||||
f"{min_loc=}, {max_loc=}, {pool.index_page_size=}"
|
||||
)
|
||||
if not tails.is_empty:
|
||||
tail_chunk_max = tails.chunk_src.to(torch.long) + torch.clamp(
|
||||
@@ -778,7 +778,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
return topk_from_pooled_history_logits(
|
||||
logits=logits,
|
||||
group_lengths=pool_lens,
|
||||
pool_size=self.index_kpool,
|
||||
kpool=self.index_kpool,
|
||||
topk=self.index_topk,
|
||||
page_table=page_table,
|
||||
topk_offsets=topk_offsets,
|
||||
@@ -960,7 +960,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
|
||||
|
||||
pool = get_token_to_kv_pool()
|
||||
page_size = pool.slots_per_page
|
||||
page_size = pool.index_page_size
|
||||
# DeepGEMM paged-MQA requires 64-token pages.
|
||||
assert page_size == 64, "only support page size 64"
|
||||
|
||||
@@ -1245,11 +1245,11 @@ class IndexerKPool(MultiPlatformOp):
|
||||
and forward_batch.extend_seq_lens_cpu is not None
|
||||
)
|
||||
|
||||
pool_size = self.index_kpool
|
||||
kpool = self.index_kpool
|
||||
page_size = metadata.attn_metadata.page_size
|
||||
slots_per_page = get_token_to_kv_pool().slots_per_page
|
||||
index_page_size = get_token_to_kv_pool().index_page_size
|
||||
token_nums = q_fp8.shape[0]
|
||||
tail_pool = pool_size - 1
|
||||
tail_pool = kpool - 1
|
||||
topk_result = torch.empty(
|
||||
(token_nums, self.index_topk + tail_pool),
|
||||
device=q_fp8.device,
|
||||
@@ -1278,7 +1278,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
continue
|
||||
|
||||
seq_len = int(forward_batch.seq_lens_cpu[i].item())
|
||||
pool_seq_len = seq_len // pool_size
|
||||
pool_seq_len = seq_len // kpool
|
||||
req_pool_idx = int(forward_batch.req_pool_indices[i].item())
|
||||
if (
|
||||
cache_write_stream is not None
|
||||
@@ -1293,7 +1293,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
local_seqlens = seq_lens_expanded[q_slice]
|
||||
if pooled_seq_lens_expanded is None:
|
||||
local_pool_lens = torch.div(
|
||||
local_seqlens, pool_size, rounding_mode="floor"
|
||||
local_seqlens, kpool, rounding_mode="floor"
|
||||
).to(torch.int32)
|
||||
else:
|
||||
local_pool_lens = pooled_seq_lens_expanded[q_slice]
|
||||
@@ -1362,10 +1362,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
i, :num_token_pages
|
||||
].contiguous()
|
||||
pool_pages = (
|
||||
curr_pool_start + slots_per_page - 1
|
||||
) // slots_per_page
|
||||
curr_pool_start + index_page_size - 1
|
||||
) // index_page_size
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
token_page_table, pool_size
|
||||
token_page_table, kpool
|
||||
)[:pool_pages].contiguous()
|
||||
k_u8 = torch.empty(
|
||||
(pool_seq_len, self.head_dim),
|
||||
@@ -1404,10 +1404,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
i, :num_token_pages
|
||||
].contiguous()
|
||||
pool_pages = (
|
||||
pool_seq_len + slots_per_page - 1
|
||||
) // slots_per_page
|
||||
pool_seq_len + index_page_size - 1
|
||||
) // index_page_size
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
token_page_table, pool_size
|
||||
token_page_table, kpool
|
||||
)[:pool_pages].contiguous()
|
||||
seq_len_t = torch.tensor(
|
||||
[pool_seq_len], dtype=torch.int32, device=q_fp8.device
|
||||
@@ -1504,7 +1504,7 @@ class IndexerKPool(MultiPlatformOp):
|
||||
|
||||
assert forward_batch.forward_mode.is_extend_without_speculative()
|
||||
|
||||
page_size = get_token_to_kv_pool().slots_per_page
|
||||
page_size = get_token_to_kv_pool().index_page_size
|
||||
assert page_size == 64, "only support page size 64"
|
||||
assert len(weights.shape) == 3
|
||||
weights = weights.squeeze(-1)
|
||||
|
||||
@@ -29,12 +29,9 @@ _is_hip = is_hip()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def kpool_metadata_fusion_supported(pool_size, page_size, topk):
|
||||
def kpool_metadata_fusion_supported(kpool, page_size, topk):
|
||||
return (
|
||||
pool_size > 1
|
||||
and page_size == 64
|
||||
and page_size % pool_size == 0
|
||||
and topk % pool_size == 0
|
||||
kpool > 1 and page_size == 64 and page_size % kpool == 0 and topk % kpool == 0
|
||||
)
|
||||
|
||||
|
||||
@@ -44,7 +41,7 @@ class DSAMetadataManagementMixin:
|
||||
def _init_kpool_metadata_fusion(self):
|
||||
requested = envs.SGLANG_EXPERIMENTAL_DSA_KPOOL_METADATA_FUSION.get()
|
||||
supported = kpool_metadata_fusion_supported(
|
||||
self.dsa_index_kpool, self.real_page_size, self.dsa_index_topk
|
||||
self.dsa_index_kpool, self.physical_page_size, self.dsa_index_topk
|
||||
)
|
||||
self.experimental_kpool_metadata_fusion = (
|
||||
requested and supported and is_cuda() and not is_hip()
|
||||
|
||||
@@ -48,23 +48,21 @@ def _kpool_cache_k_offsets(
|
||||
return page * BUF_NUMEL_PER_PAGE + slot * HEAD_DIM + cols
|
||||
|
||||
|
||||
def kpool_max_closed_pools(num_draft_tokens: int, pool_size: int) -> int:
|
||||
return (num_draft_tokens + pool_size - 1) // pool_size
|
||||
def kpool_max_closed_pools(num_draft_tokens: int, kpool: int) -> int:
|
||||
return (num_draft_tokens + kpool - 1) // kpool
|
||||
|
||||
|
||||
def build_pooled_page_table_64(
|
||||
page_table_64: torch.Tensor,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
) -> torch.Tensor:
|
||||
# Advanced indexing is required: a (1, 1) strided slice can remain non-unit-
|
||||
# stride even after contiguous(), which DeepGEMM rejects.
|
||||
assert BLOCK_SIZE_K % pool_size == 0, (
|
||||
f"pool_size ({pool_size}) must divide page_size ({BLOCK_SIZE_K})"
|
||||
assert BLOCK_SIZE_K % kpool == 0, (
|
||||
f"kpool ({kpool}) must divide page_size ({BLOCK_SIZE_K})"
|
||||
)
|
||||
idx = torch.arange(
|
||||
0, page_table_64.shape[-1], pool_size, device=page_table_64.device
|
||||
)
|
||||
return page_table_64[..., idx] // pool_size
|
||||
idx = torch.arange(0, page_table_64.shape[-1], kpool, device=page_table_64.device)
|
||||
return page_table_64[..., idx] // kpool
|
||||
|
||||
|
||||
def gather_index_k_scale_prefix_into(
|
||||
@@ -79,7 +77,7 @@ def gather_index_k_scale_prefix_into(
|
||||
assert page_indices.dtype in (torch.int32, torch.int64)
|
||||
assert k_out.dtype == torch.uint8
|
||||
assert scale_out.dtype == torch.float32
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert pool.index_page_size == BLOCK_SIZE_K
|
||||
assert k_out.shape[0] >= seq_len
|
||||
assert k_out.shape[1] == INDEX_HEAD_DIM
|
||||
assert scale_out.shape[0] >= seq_len
|
||||
@@ -96,11 +94,11 @@ def gather_index_k_scale_prefix_into(
|
||||
page_indices,
|
||||
k_out,
|
||||
scale_out,
|
||||
PAGE_SIZE=pool.slots_per_page,
|
||||
PAGE_SIZE=pool.index_page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * INDEX_HEAD_DIM,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
)
|
||||
|
||||
@@ -154,10 +152,10 @@ def kpool_build_ragged_layout(
|
||||
cu_q_len_excl: torch.Tensor,
|
||||
ragged_q_len: torch.Tensor,
|
||||
pooled_seq_lens_expanded: torch.Tensor,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
total_pool_pages: int,
|
||||
total_q: int,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
device = full_page_table.device
|
||||
n_rag = cu_pages_excl.shape[0]
|
||||
@@ -181,8 +179,8 @@ def kpool_build_ragged_layout(
|
||||
q_ks,
|
||||
q_ke,
|
||||
max_pool_pages,
|
||||
slots_per_page,
|
||||
pool_size,
|
||||
index_page_size,
|
||||
kpool,
|
||||
BLOCK_PAGE=128,
|
||||
BLOCK_Q=128,
|
||||
)
|
||||
@@ -201,7 +199,7 @@ def _kpool_build_ragged_layout_kernel(
|
||||
q_ks_ptr,
|
||||
q_ke_ptr,
|
||||
MAX_POOL_PAGES,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
INDEX_PAGE_SIZE: tl.constexpr,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
BLOCK_PAGE: tl.constexpr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
@@ -211,7 +209,7 @@ def _kpool_build_ragged_layout_kernel(
|
||||
n_pages = tl.load(ragged_pool_pages_ptr + k)
|
||||
q_start = tl.load(cu_q_len_excl_ptr + k)
|
||||
q_count = tl.load(ragged_q_len_ptr + k)
|
||||
ks_val = page_start * SLOTS_PER_PAGE
|
||||
ks_val = page_start * INDEX_PAGE_SIZE
|
||||
|
||||
for p_off in tl.range(0, BLOCK_PAGE * tl.cdiv(n_pages, BLOCK_PAGE), BLOCK_PAGE):
|
||||
p_offs = p_off + tl.arange(0, BLOCK_PAGE)
|
||||
@@ -234,7 +232,7 @@ def _kpool_build_ragged_layout_kernel(
|
||||
plen = tl.load(pooled_seq_lens_ptr + q_start + q_offs, mask=q_mask, other=0)
|
||||
ke_val = tl.minimum(
|
||||
ks_val + plen,
|
||||
(page_start + n_pages) * SLOTS_PER_PAGE,
|
||||
(page_start + n_pages) * INDEX_PAGE_SIZE,
|
||||
)
|
||||
tl.store(
|
||||
q_ks_ptr + q_start + q_offs,
|
||||
@@ -255,13 +253,13 @@ def _prep_update_kpool_write_plan_launch(
|
||||
pool_seqlens_per_q_out: Optional[torch.Tensor],
|
||||
seqlens_per_q_out: Optional[torch.Tensor],
|
||||
*,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
num_draft_tokens: int,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
):
|
||||
# Eager and captured replay must share one launch spec, including
|
||||
# N > pool_size where one write closes multiple pools.
|
||||
max_closed_pools = kpool_max_closed_pools(num_draft_tokens, pool_size)
|
||||
# N > kpool where one write closes multiple pools.
|
||||
max_closed_pools = kpool_max_closed_pools(num_draft_tokens, kpool)
|
||||
bs = write_start.shape[0]
|
||||
|
||||
assert write_loc_out.shape == (bs, max_closed_pools), write_loc_out.shape
|
||||
@@ -292,9 +290,9 @@ def _prep_update_kpool_write_plan_launch(
|
||||
write_loc_out.stride(0),
|
||||
)
|
||||
constexprs = dict(
|
||||
POOL_SIZE=pool_size,
|
||||
POOL_SIZE=kpool,
|
||||
N=num_draft_tokens,
|
||||
SLOTS_PER_PAGE=slots_per_page,
|
||||
INDEX_PAGE_SIZE=index_page_size,
|
||||
MAX_CLOSED_POOLS=max_closed_pools,
|
||||
HAS_PER_Q=has_per_q_outputs,
|
||||
)
|
||||
@@ -312,9 +310,9 @@ def update_kpool_write_plan_cuda_graph(
|
||||
pool_seqlens_per_q_out: Optional[torch.Tensor],
|
||||
seqlens_per_q_out: Optional[torch.Tensor],
|
||||
*,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
num_draft_tokens: int,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
) -> None:
|
||||
if write_start.shape[0] == 0:
|
||||
return
|
||||
@@ -328,9 +326,9 @@ def update_kpool_write_plan_cuda_graph(
|
||||
write_loc_out,
|
||||
pool_seqlens_per_q_out,
|
||||
seqlens_per_q_out,
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
_update_kpool_write_plan_kernel[(bs,)](*args, **constexprs)
|
||||
|
||||
@@ -351,7 +349,7 @@ def _update_kpool_write_plan_kernel(
|
||||
write_loc_out_stride_0,
|
||||
POOL_SIZE: tl.constexpr,
|
||||
N: tl.constexpr,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
INDEX_PAGE_SIZE: tl.constexpr,
|
||||
MAX_CLOSED_POOLS: tl.constexpr,
|
||||
HAS_PER_Q: tl.constexpr,
|
||||
):
|
||||
@@ -372,7 +370,7 @@ def _update_kpool_write_plan_kernel(
|
||||
tl.store(tail_logical_start_out_ptr + b, (base_pool * POOL_SIZE).to(tl.int32))
|
||||
for p in tl.static_range(0, MAX_CLOSED_POOLS):
|
||||
pool_id = base_pool + p
|
||||
pool_page_group = pool_id // SLOTS_PER_PAGE
|
||||
pool_page_group = pool_id // INDEX_PAGE_SIZE
|
||||
token_page_row = pool_page_group * POOL_SIZE
|
||||
token_page_row = tl.minimum(
|
||||
tl.maximum(token_page_row, 0), real_page_table_cols - 1
|
||||
@@ -387,7 +385,7 @@ def _update_kpool_write_plan_kernel(
|
||||
).to(tl.int64)
|
||||
// POOL_SIZE
|
||||
)
|
||||
write_loc = packed_page * SLOTS_PER_PAGE + (pool_id % SLOTS_PER_PAGE)
|
||||
write_loc = packed_page * INDEX_PAGE_SIZE + (pool_id % INDEX_PAGE_SIZE)
|
||||
tl.store(
|
||||
write_loc_out_ptr + b * write_loc_out_stride_0 + p,
|
||||
write_loc.to(tl.int64),
|
||||
@@ -397,45 +395,45 @@ def _update_kpool_write_plan_kernel(
|
||||
def compute_pooled_write_locs(
|
||||
page_table_64: torch.Tensor,
|
||||
pool_ids: torch.Tensor,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
) -> torch.Tensor:
|
||||
assert page_table_64.ndim == 1
|
||||
pool_ids = pool_ids.to(torch.int64)
|
||||
pool_page_group = torch.div(pool_ids, BLOCK_SIZE_K, rounding_mode="floor")
|
||||
token_page_row = pool_page_group * pool_size
|
||||
token_page_row = pool_page_group * kpool
|
||||
packed_page = (
|
||||
page_table_64.index_select(0, token_page_row.to(torch.int64)).to(torch.int64)
|
||||
// pool_size
|
||||
// kpool
|
||||
)
|
||||
return packed_page * BLOCK_SIZE_K + torch.remainder(pool_ids, BLOCK_SIZE_K)
|
||||
|
||||
|
||||
def history_group_budget_for_topk(topk: int, pool_size: int) -> int:
|
||||
assert topk % pool_size == 0
|
||||
return topk // pool_size
|
||||
def history_group_budget_for_topk(topk: int, kpool: int) -> int:
|
||||
assert topk % kpool == 0
|
||||
return topk // kpool
|
||||
|
||||
|
||||
def expand_pooled_groups_to_topk(
|
||||
group_ids: torch.Tensor,
|
||||
group_valid: torch.Tensor,
|
||||
topk: int,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert group_ids.ndim == 2
|
||||
assert group_valid.shape == group_ids.shape
|
||||
assert topk % pool_size == 0
|
||||
assert group_ids.shape[1] == history_group_budget_for_topk(topk, pool_size)
|
||||
assert topk % kpool == 0
|
||||
assert group_ids.shape[1] == history_group_budget_for_topk(topk, kpool)
|
||||
assert page_table is None or topk_offsets is None
|
||||
|
||||
device = group_ids.device
|
||||
offsets = torch.arange(pool_size, device=device, dtype=torch.int64)
|
||||
token_ids = group_ids.to(torch.int64).unsqueeze(-1) * pool_size + offsets
|
||||
offsets = torch.arange(kpool, device=device, dtype=torch.int64)
|
||||
token_ids = group_ids.to(torch.int64).unsqueeze(-1) * kpool + offsets
|
||||
token_ids = token_ids.reshape(group_ids.shape[0], topk)
|
||||
valid = (
|
||||
group_valid.unsqueeze(-1)
|
||||
.expand(-1, -1, pool_size)
|
||||
.expand(-1, -1, kpool)
|
||||
.reshape(group_ids.shape[0], topk)
|
||||
)
|
||||
|
||||
@@ -460,7 +458,7 @@ def append_kpool_tail_to_topk(
|
||||
topk_result: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
pool_lens: torch.Tensor,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
@@ -470,7 +468,7 @@ def append_kpool_tail_to_topk(
|
||||
assert seq_lens.shape[0] == topk_result.shape[0]
|
||||
assert pool_lens.shape[0] == topk_result.shape[0]
|
||||
|
||||
tail_pool = pool_size - 1
|
||||
tail_pool = kpool - 1
|
||||
if tail_pool == 0:
|
||||
return topk_result
|
||||
|
||||
@@ -516,7 +514,7 @@ def append_kpool_tail_to_topk(
|
||||
N_COLS=n_cols,
|
||||
OUT_COLS=out_cols,
|
||||
PAGE_TABLE_COLS=page_table_cols,
|
||||
POOL_SIZE=pool_size,
|
||||
POOL_SIZE=kpool,
|
||||
HAS_PAGE_TABLE=has_page_table,
|
||||
HAS_TOPK_OFFSETS=has_topk_offsets,
|
||||
BLOCK_COLS=block_cols,
|
||||
@@ -591,7 +589,7 @@ def _append_kpool_tail_to_topk_kernel(
|
||||
def topk_from_pooled_history_logits(
|
||||
logits: torch.Tensor,
|
||||
group_lengths: torch.Tensor,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
topk: int,
|
||||
page_table: torch.Tensor | None = None,
|
||||
topk_offsets: torch.Tensor | None = None,
|
||||
@@ -604,12 +602,12 @@ def topk_from_pooled_history_logits(
|
||||
assert group_lengths.ndim == 1
|
||||
assert logits.shape[0] == group_lengths.shape[0]
|
||||
assert topk > 0
|
||||
assert topk % pool_size == 0
|
||||
assert topk % kpool == 0
|
||||
assert out_rows is None or out_rows >= logits.shape[0]
|
||||
assert page_table_row_index is None or page_table is not None
|
||||
|
||||
_, cols = logits.shape
|
||||
group_topk = history_group_budget_for_topk(topk, pool_size)
|
||||
group_topk = history_group_budget_for_topk(topk, kpool)
|
||||
if topk_offsets is not None and topk_offsets.ndim == 2:
|
||||
assert topk_offsets.shape[1] == 1
|
||||
topk_offsets = topk_offsets.squeeze(1)
|
||||
@@ -618,7 +616,7 @@ def topk_from_pooled_history_logits(
|
||||
raise NotImplementedError(
|
||||
"index_kpool topk only supports pooled group_topk in "
|
||||
f"(128, 160, 192, 224, 256, 512, 2048), got {group_topk} "
|
||||
f"(topk={topk}, pool_size={pool_size})."
|
||||
f"(topk={topk}, kpool={kpool})."
|
||||
)
|
||||
if not logits.is_cuda or logits.dtype != torch.float32:
|
||||
raise NotImplementedError(
|
||||
@@ -634,7 +632,7 @@ def topk_from_pooled_history_logits(
|
||||
result = fast_kpool_topk_transform_fused(
|
||||
score=logits,
|
||||
lengths=group_lengths.to(torch.int32),
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
topk=topk,
|
||||
page_table=page_table,
|
||||
topk_indices_offset=topk_offsets,
|
||||
@@ -674,7 +672,7 @@ def topk_from_pooled_history_logits(
|
||||
selected_groups.contiguous(),
|
||||
group_valid,
|
||||
topk=topk,
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
page_table=page_table,
|
||||
topk_offsets=topk_offsets,
|
||||
)
|
||||
@@ -685,7 +683,7 @@ def topk_from_pooled_history_logits(
|
||||
expanded,
|
||||
seq_lens=seq_lens,
|
||||
pool_lens=group_lengths,
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
page_table=page_table,
|
||||
topk_offsets=topk_offsets,
|
||||
)
|
||||
@@ -718,7 +716,7 @@ def kpool_softmax_rotate_write_cache(
|
||||
assert slot_score.dtype in KPOOL_SCORE_DTYPES
|
||||
assert ape.dtype == torch.float32
|
||||
assert buf.dtype == torch.uint8
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert pool.index_page_size == BLOCK_SIZE_K
|
||||
assert pool.index_head_dim == INDEX_HEAD_DIM
|
||||
assert loc.dtype == torch.int64
|
||||
assert write_cache or return_compressed
|
||||
@@ -777,12 +775,12 @@ def kpool_softmax_rotate_write_cache(
|
||||
slot_score.stride(0),
|
||||
slot_score.stride(1),
|
||||
ape.stride(0),
|
||||
PAGE_SIZE=pool.slots_per_page,
|
||||
PAGE_SIZE=pool.index_page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=slot_k.shape[1],
|
||||
HEAD_DIM=slot_k.shape[2],
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * pool.index_head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
RETURN_COMPRESSED=return_compressed,
|
||||
@@ -822,7 +820,7 @@ def kpool_decode_update_and_maybe_write_cache(
|
||||
assert slot_score.dtype == tail_score.dtype
|
||||
assert ape.dtype == torch.float32
|
||||
assert buf.dtype == torch.uint8
|
||||
assert pool.slots_per_page == BLOCK_SIZE_K
|
||||
assert pool.index_page_size == BLOCK_SIZE_K
|
||||
assert pool.index_head_dim == INDEX_HEAD_DIM
|
||||
assert tail_k.is_contiguous()
|
||||
assert tail_score.is_contiguous()
|
||||
@@ -878,10 +876,10 @@ def kpool_decode_update_and_maybe_write_cache(
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=tail_k.shape[2],
|
||||
BLOCK_TABLE_COLS=block_tables.shape[1],
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.slots_per_page * pool.index_head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=pool.index_page_size * pool.index_head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
BLOCK_D=triton.next_power_of_2(tail_k.shape[2]),
|
||||
SLOTS_PER_PAGE=pool.slots_per_page,
|
||||
INDEX_PAGE_SIZE=pool.index_page_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1063,7 +1061,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
INDEX_PAGE_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
@@ -1163,7 +1161,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
quantized = tl.minimum(tl.maximum(quantized, fp8_min), fp8_max)
|
||||
|
||||
pool_id = safe_pos // POOL_SIZE
|
||||
pool_page_group = pool_id // SLOTS_PER_PAGE
|
||||
pool_page_group = pool_id // INDEX_PAGE_SIZE
|
||||
token_page_row = pool_page_group * POOL_SIZE
|
||||
token_page_row = tl.minimum(tl.maximum(token_page_row, 0), BLOCK_TABLE_COLS - 1)
|
||||
packed_page = tl.load(
|
||||
@@ -1172,7 +1170,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
|
||||
+ token_page_row * block_tables_stride_1,
|
||||
)
|
||||
loc_page_index = packed_page.to(tl.int64) // POOL_SIZE
|
||||
loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE
|
||||
loc_token_offset_in_page = pool_id % INDEX_PAGE_SIZE
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
@@ -1243,7 +1241,7 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
|
||||
ROUND_SCALE: tl.constexpr,
|
||||
HAS_WRITE_MASK: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
INDEX_PAGE_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
if HAS_WRITE_MASK:
|
||||
@@ -1285,8 +1283,8 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
|
||||
quantized, scale = _hadamard_quantize_fp8(acc, denom, ROUND_SCALE)
|
||||
|
||||
loc = tl.load(loc_ptr + row)
|
||||
loc_page_index = loc // SLOTS_PER_PAGE
|
||||
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
|
||||
loc_page_index = loc // INDEX_PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % INDEX_PAGE_SIZE
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
@@ -1321,7 +1319,7 @@ def kpool_assemble_softmax_rotate_write_cache(
|
||||
write_mask: torch.Tensor | None = None,
|
||||
round_scale: bool = False,
|
||||
) -> None:
|
||||
pool_size = pool.index_kpool
|
||||
kpool = pool.index_kpool
|
||||
n_pools = req_pool_idx.shape[0]
|
||||
if n_pools == 0:
|
||||
return
|
||||
@@ -1339,7 +1337,7 @@ def kpool_assemble_softmax_rotate_write_cache(
|
||||
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
slots_per_page = pool.slots_per_page
|
||||
index_page_size = pool.index_page_size
|
||||
|
||||
_kpool_assemble_softmax_rotate_write_cache_kernel[(n_pools,)](
|
||||
buf_fp8,
|
||||
@@ -1361,14 +1359,14 @@ def kpool_assemble_softmax_rotate_write_cache(
|
||||
ape.stride(0),
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
POOL_SIZE=pool_size,
|
||||
POOL_SIZE=kpool,
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=slots_per_page * INDEX_HEAD_DIM,
|
||||
S_OFFSET_NBYTES_IN_PAGE=index_page_size * INDEX_HEAD_DIM,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_WRITE_MASK=has_write_mask,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
SLOTS_PER_PAGE=slots_per_page,
|
||||
INDEX_PAGE_SIZE=index_page_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1383,14 +1381,14 @@ def scatter_kpool_tail_updates(
|
||||
chunk_src_start: torch.Tensor,
|
||||
n_write: torch.Tensor,
|
||||
) -> None:
|
||||
pool_size = pool.index_kpool
|
||||
kpool = pool.index_kpool
|
||||
n_rows = req_pool_idx.shape[0]
|
||||
if n_rows == 0:
|
||||
return
|
||||
|
||||
chunk_k = chunk_k.contiguous()
|
||||
chunk_score = chunk_score.contiguous()
|
||||
_scatter_kpool_tail_updates_kernel[(n_rows, pool_size)](
|
||||
_scatter_kpool_tail_updates_kernel[(n_rows, kpool)](
|
||||
chunk_k,
|
||||
chunk_score,
|
||||
tail_k,
|
||||
@@ -1402,7 +1400,7 @@ def scatter_kpool_tail_updates(
|
||||
chunk_k.stride(0),
|
||||
tail_k.stride(0),
|
||||
tail_k.stride(1),
|
||||
POOL_SIZE=pool_size,
|
||||
POOL_SIZE=kpool,
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
@@ -1458,7 +1456,7 @@ def _pack_pool_slots_to_payload_kernel(
|
||||
locs_ptr,
|
||||
payload_ptr,
|
||||
payload_bytes: tl.constexpr,
|
||||
slots_per_page: tl.constexpr,
|
||||
index_page_size: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
page_bytes: tl.constexpr,
|
||||
scale_region_off: tl.constexpr,
|
||||
@@ -1466,8 +1464,8 @@ def _pack_pool_slots_to_payload_kernel(
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
loc = tl.load(locs_ptr + row).to(tl.int64)
|
||||
page = loc // slots_per_page
|
||||
slot = loc % slots_per_page
|
||||
page = loc // index_page_size
|
||||
slot = loc % index_page_size
|
||||
page_base = page * page_bytes
|
||||
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
@@ -1490,7 +1488,7 @@ def _select_and_scatter_pool_slots_kernel(
|
||||
buf_ptr,
|
||||
cp_rank: tl.constexpr,
|
||||
payload_bytes: tl.constexpr,
|
||||
slots_per_page: tl.constexpr,
|
||||
index_page_size: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
page_bytes: tl.constexpr,
|
||||
scale_region_off: tl.constexpr,
|
||||
@@ -1503,8 +1501,8 @@ def _select_and_scatter_pool_slots_kernel(
|
||||
return
|
||||
|
||||
loc = tl.load(locs_ptr + row).to(tl.int64)
|
||||
page = loc // slots_per_page
|
||||
slot = loc % slots_per_page
|
||||
page = loc // index_page_size
|
||||
slot = loc % index_page_size
|
||||
page_base = page * page_bytes
|
||||
recv_row_base = (owner * n_total + row) * payload_bytes
|
||||
|
||||
@@ -1526,7 +1524,7 @@ def all_gather_and_scatter_pool_slots(
|
||||
owner_rank: torch.Tensor,
|
||||
cp_size: int,
|
||||
cp_rank: int,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
) -> None:
|
||||
from sglang.srt.layers.dp_attention import attn_cp_all_gather_into_tensor
|
||||
|
||||
@@ -1537,7 +1535,7 @@ def all_gather_and_scatter_pool_slots(
|
||||
|
||||
head_dim = INDEX_HEAD_DIM
|
||||
payload_bytes = head_dim + 4
|
||||
scale_region_off = slots_per_page * head_dim
|
||||
scale_region_off = index_page_size * head_dim
|
||||
page_bytes = buf.shape[1]
|
||||
device = buf.device
|
||||
|
||||
@@ -1549,7 +1547,7 @@ def all_gather_and_scatter_pool_slots(
|
||||
local_locs,
|
||||
send_payload,
|
||||
payload_bytes=payload_bytes,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
head_dim=head_dim,
|
||||
page_bytes=page_bytes,
|
||||
scale_region_off=scale_region_off,
|
||||
@@ -1570,7 +1568,7 @@ def all_gather_and_scatter_pool_slots(
|
||||
buf,
|
||||
cp_rank=cp_rank,
|
||||
payload_bytes=payload_bytes,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
head_dim=head_dim,
|
||||
page_bytes=page_bytes,
|
||||
scale_region_off=scale_region_off,
|
||||
@@ -1605,7 +1603,7 @@ def _kpool_write_tail_and_maybe_compress_kernel(
|
||||
TAIL_SIZE: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
SLOTS_PER_PAGE: tl.constexpr,
|
||||
INDEX_PAGE_SIZE: tl.constexpr,
|
||||
BUF_NUMEL_PER_PAGE: tl.constexpr,
|
||||
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
|
||||
PRESHUFFLE_TILE: tl.constexpr,
|
||||
@@ -1671,8 +1669,8 @@ def _kpool_write_tail_and_maybe_compress_kernel(
|
||||
|
||||
quantized, scale = _hadamard_quantize_fp8(acc, denom, ROUND_SCALE)
|
||||
loc = tl.load(write_loc_ptr + b * write_loc_stride_0 + p)
|
||||
loc_page_index = loc // SLOTS_PER_PAGE
|
||||
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
|
||||
loc_page_index = loc // INDEX_PAGE_SIZE
|
||||
loc_token_offset_in_page = loc % INDEX_PAGE_SIZE
|
||||
out_k_offsets = _kpool_cache_k_offsets(
|
||||
loc_page_index,
|
||||
loc_token_offset_in_page,
|
||||
@@ -1739,7 +1737,7 @@ def kpool_write_tail_and_maybe_compress(
|
||||
if effective_n_per_batch is not None:
|
||||
effective_n_per_batch = effective_n_per_batch.contiguous()
|
||||
|
||||
slots_per_page = pool.slots_per_page
|
||||
index_page_size = pool.index_page_size
|
||||
buf_fp8 = buf.view(torch.float8_e4m3fn)
|
||||
buf_fp32 = buf.view(torch.float32)
|
||||
_kpool_write_tail_and_maybe_compress_kernel[(bs,)](
|
||||
@@ -1767,10 +1765,10 @@ def kpool_write_tail_and_maybe_compress(
|
||||
TAIL_SIZE=tail_k.shape[1],
|
||||
HEAD_DIM=INDEX_HEAD_DIM,
|
||||
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
|
||||
SLOTS_PER_PAGE=slots_per_page,
|
||||
INDEX_PAGE_SIZE=index_page_size,
|
||||
BUF_NUMEL_PER_PAGE=buf.shape[1],
|
||||
PRESHUFFLE_TILE=_preshuffle_tile(),
|
||||
S_OFFSET_NBYTES_IN_PAGE=slots_per_page * pool.index_head_dim,
|
||||
S_OFFSET_NBYTES_IN_PAGE=index_page_size * pool.index_head_dim,
|
||||
ROUND_SCALE=round_scale,
|
||||
HAS_EFFECTIVE_N=effective_n_per_batch is not None,
|
||||
MAX_CLOSED_POOLS=max_closed_pools,
|
||||
|
||||
@@ -117,8 +117,8 @@ class KPoolWritePlan:
|
||||
effective_n_per_batch: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def _is_kpool_layout_enabled(pool_size: int, real_page_size: int) -> bool:
|
||||
return pool_size > 1 and real_page_size == 64 and real_page_size % pool_size == 0
|
||||
def _is_kpool_layout_enabled(kpool: int, physical_page_size: int) -> bool:
|
||||
return kpool > 1 and physical_page_size == 64 and physical_page_size % kpool == 0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -149,11 +149,11 @@ class _KPoolDecompose(NamedTuple):
|
||||
tail_n_write: int
|
||||
|
||||
|
||||
def _decompose_compress(start: int, length: int, pool_size: int) -> _KPoolDecompose:
|
||||
first_slot = start % pool_size
|
||||
base_pool = start // pool_size
|
||||
n_pool = (start + length) // pool_size - base_pool
|
||||
consumed = max(0, n_pool * pool_size - first_slot)
|
||||
def _decompose_compress(start: int, length: int, kpool: int) -> _KPoolDecompose:
|
||||
first_slot = start % kpool
|
||||
base_pool = start // kpool
|
||||
n_pool = (start + length) // kpool - base_pool
|
||||
consumed = max(0, n_pool * kpool - first_slot)
|
||||
tail_n = length - consumed
|
||||
return _KPoolDecompose(
|
||||
first_slot=first_slot,
|
||||
@@ -165,7 +165,7 @@ def _decompose_compress(start: int, length: int, pool_size: int) -> _KPoolDecomp
|
||||
|
||||
def _append_compress_rows(
|
||||
plan: _KPoolCpuPlan,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
batch_size: int,
|
||||
extend_seq_lens_cpu: List[int],
|
||||
seq_lens_cpu: List[int],
|
||||
@@ -178,7 +178,7 @@ def _append_compress_rows(
|
||||
|
||||
seq_len = seq_lens_cpu[i]
|
||||
req = req_pool_indices_cpu[i]
|
||||
d = _decompose_compress(seq_len - q_len, q_len, pool_size)
|
||||
d = _decompose_compress(seq_len - q_len, q_len, kpool)
|
||||
|
||||
if d.n_pool > 0:
|
||||
plan.pool_batch_idx.extend([i] * d.n_pool)
|
||||
@@ -186,16 +186,16 @@ def _append_compress_rows(
|
||||
plan.pool_pool_id.extend(range(d.base_pool, d.base_pool + d.n_pool))
|
||||
plan.pool_n_from_tail.append(d.first_slot)
|
||||
plan.pool_n_from_tail.extend([0] * (d.n_pool - 1))
|
||||
bulk_start = q_offset + pool_size - d.first_slot
|
||||
bulk_start = q_offset + kpool - d.first_slot
|
||||
plan.pool_chunk_src.append(q_offset)
|
||||
plan.pool_chunk_src.extend(
|
||||
range(bulk_start, bulk_start + (d.n_pool - 1) * pool_size, pool_size)
|
||||
range(bulk_start, bulk_start + (d.n_pool - 1) * kpool, kpool)
|
||||
)
|
||||
plan.pool_tail_logical_base.extend(
|
||||
range(
|
||||
d.base_pool * pool_size,
|
||||
(d.base_pool + d.n_pool) * pool_size,
|
||||
pool_size,
|
||||
d.base_pool * kpool,
|
||||
(d.base_pool + d.n_pool) * kpool,
|
||||
kpool,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -211,8 +211,8 @@ def _append_compress_rows(
|
||||
|
||||
def _append_local_rows(
|
||||
plan: _KPoolCpuPlan,
|
||||
pool_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
index_page_size: int,
|
||||
local_extend_seq_lens_cpu: List[int],
|
||||
local_seq_lens_cpu: List[int],
|
||||
) -> None:
|
||||
@@ -222,8 +222,8 @@ def _append_local_rows(
|
||||
):
|
||||
assert q_len > 0, f"local_extend_seq_lens_cpu has non-positive {q_len = }"
|
||||
plan.ragged_q_len.append(q_len)
|
||||
pool_seq_len = seq_len // pool_size
|
||||
pool_pages_i = (pool_seq_len + slots_per_page - 1) // slots_per_page
|
||||
pool_seq_len = seq_len // kpool
|
||||
pool_pages_i = (pool_seq_len + index_page_size - 1) // index_page_size
|
||||
plan.ragged_pool_pages.append(pool_pages_i)
|
||||
plan.cu_pages_excl.append(plan.total_pool_pages)
|
||||
plan.cu_q_len_excl.append(q_offset)
|
||||
@@ -233,8 +233,8 @@ def _append_local_rows(
|
||||
|
||||
def _kpool_cpu_plan(
|
||||
forward_batch: ForwardBatch,
|
||||
pool_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
index_page_size: int,
|
||||
*,
|
||||
local_extend_seq_lens_cpu: Optional[List[int]] = None,
|
||||
local_seq_lens_cpu: Optional[List[int]] = None,
|
||||
@@ -252,7 +252,7 @@ def _kpool_cpu_plan(
|
||||
|
||||
_append_compress_rows(
|
||||
plan,
|
||||
pool_size,
|
||||
kpool,
|
||||
forward_batch.batch_size,
|
||||
extend_seq_lens_cpu,
|
||||
seq_lens_cpu,
|
||||
@@ -265,8 +265,8 @@ def _kpool_cpu_plan(
|
||||
|
||||
_append_local_rows(
|
||||
plan,
|
||||
pool_size,
|
||||
slots_per_page,
|
||||
kpool,
|
||||
index_page_size,
|
||||
local_extend_seq_lens_cpu,
|
||||
local_seq_lens_cpu,
|
||||
)
|
||||
@@ -280,8 +280,8 @@ def _kpool_plan_to_gpu(
|
||||
local_real_page_table: torch.Tensor,
|
||||
local_seqlens_expanded: torch.Tensor,
|
||||
local_req_pool_indices: torch.Tensor,
|
||||
pool_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
index_page_size: int,
|
||||
topk_transform_method: TopkTransformMethod,
|
||||
) -> KPoolExtendPlan:
|
||||
from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod
|
||||
@@ -292,7 +292,7 @@ def _kpool_plan_to_gpu(
|
||||
n_rag = len(cpu.ragged_q_len)
|
||||
|
||||
total_pool_pages = cpu.total_pool_pages
|
||||
ragged_total_k_rows = total_pool_pages * slots_per_page
|
||||
ragged_total_k_rows = total_pool_pages * index_page_size
|
||||
|
||||
need_paged = (
|
||||
topk_transform_method == TopkTransformMethod.PAGED
|
||||
@@ -370,21 +370,21 @@ def _kpool_plan_to_gpu(
|
||||
|
||||
if n_pool > 0:
|
||||
pool_page_group = torch.div(
|
||||
pool_pool_id_t, slots_per_page, rounding_mode="floor"
|
||||
pool_pool_id_t, index_page_size, rounding_mode="floor"
|
||||
)
|
||||
token_page_row = pool_page_group * pool_size
|
||||
token_page_row = pool_page_group * kpool
|
||||
packed_page = (
|
||||
full_real_page_table[pool_batch_idx_t, token_page_row].to(torch.int64)
|
||||
// pool_size
|
||||
// kpool
|
||||
)
|
||||
pool_write_locs = packed_page * slots_per_page + torch.remainder(
|
||||
pool_pool_id_t, slots_per_page
|
||||
pool_write_locs = packed_page * index_page_size + torch.remainder(
|
||||
pool_pool_id_t, index_page_size
|
||||
)
|
||||
else:
|
||||
pool_write_locs = torch.empty((0,), dtype=torch.int64, device=device)
|
||||
|
||||
pooled_seq_lens_expanded = torch.div(
|
||||
local_seqlens_expanded, pool_size, rounding_mode="floor"
|
||||
local_seqlens_expanded, kpool, rounding_mode="floor"
|
||||
).to(torch.int32)
|
||||
|
||||
if n_rag > 0:
|
||||
@@ -399,10 +399,10 @@ def _kpool_plan_to_gpu(
|
||||
cu_q_len_excl=cu_q_len_excl_t,
|
||||
ragged_q_len=ragged_q_len_t,
|
||||
pooled_seq_lens_expanded=pooled_seq_lens_expanded,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
total_pool_pages=total_pool_pages,
|
||||
total_q=pooled_seq_lens_expanded.shape[0],
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
)
|
||||
else:
|
||||
empty_i32_dev = torch.empty((0,), dtype=torch.int32, device=device)
|
||||
@@ -487,9 +487,9 @@ def init_kpool_extend_metadata(
|
||||
metadata: DSAMetadata,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
index_page_size: int,
|
||||
topk_transform_method: TopkTransformMethod,
|
||||
full_real_page_table: torch.Tensor,
|
||||
full_seqlens_expanded: torch.Tensor,
|
||||
@@ -502,7 +502,7 @@ def init_kpool_extend_metadata(
|
||||
mode = forward_batch.forward_mode
|
||||
is_extend_like = mode.is_extend_without_speculative() or mode.is_draft_extend_v2()
|
||||
if (
|
||||
not _is_kpool_layout_enabled(pool_size, real_page_size)
|
||||
not _is_kpool_layout_enabled(kpool, physical_page_size)
|
||||
or not is_extend_like
|
||||
or forward_batch.extend_seq_lens_cpu is None
|
||||
or forward_batch.seq_lens_cpu is None
|
||||
@@ -518,8 +518,8 @@ def init_kpool_extend_metadata(
|
||||
|
||||
cpu = _kpool_cpu_plan(
|
||||
forward_batch,
|
||||
pool_size,
|
||||
slots_per_page,
|
||||
kpool,
|
||||
index_page_size,
|
||||
local_extend_seq_lens_cpu=local_extend_seq_lens_cpu,
|
||||
local_seq_lens_cpu=local_seq_lens_cpu,
|
||||
)
|
||||
@@ -530,8 +530,8 @@ def init_kpool_extend_metadata(
|
||||
local_real_page_table,
|
||||
local_seqlens_expanded,
|
||||
local_req_pool_indices,
|
||||
pool_size,
|
||||
slots_per_page,
|
||||
kpool,
|
||||
index_page_size,
|
||||
topk_transform_method,
|
||||
)
|
||||
return dataclasses.replace(metadata, kpool_extend_plan=plan)
|
||||
@@ -558,7 +558,7 @@ def _get_deep_gemm():
|
||||
def _compute_pool_schedule_metadata(
|
||||
pool_seqlens: torch.Tensor,
|
||||
*,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
) -> Optional[torch.Tensor]:
|
||||
if not is_cuda():
|
||||
return None
|
||||
@@ -567,7 +567,7 @@ def _compute_pool_schedule_metadata(
|
||||
return None
|
||||
return deep_gemm.get_paged_mqa_logits_metadata(
|
||||
pool_seqlens.contiguous().view(-1, 1).clamp(min=1),
|
||||
slots_per_page,
|
||||
index_page_size,
|
||||
deep_gemm.get_num_sms(),
|
||||
)
|
||||
|
||||
@@ -577,42 +577,40 @@ def init_pooled_paged_mqa_metadata(
|
||||
seqlens_32: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
index_page_size: int,
|
||||
build_schedule_metadata: bool = True,
|
||||
) -> DSAMetadata:
|
||||
if (
|
||||
not _is_kpool_layout_enabled(pool_size, real_page_size)
|
||||
not _is_kpool_layout_enabled(kpool, physical_page_size)
|
||||
or not is_cuda()
|
||||
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
|
||||
):
|
||||
return metadata
|
||||
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
metadata.real_page_table, pool_size
|
||||
metadata.real_page_table, kpool
|
||||
).contiguous()
|
||||
if forward_mode.is_target_verify():
|
||||
return dataclasses.replace(
|
||||
metadata,
|
||||
pooled_index_kpool=pool_size,
|
||||
pooled_index_kpool=kpool,
|
||||
pooled_real_page_table=pooled_page_table,
|
||||
)
|
||||
|
||||
pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to(
|
||||
torch.int32
|
||||
)
|
||||
pool_seqlens = torch.div(seqlens_32, kpool, rounding_mode="floor").to(torch.int32)
|
||||
schedule = (
|
||||
_compute_pool_schedule_metadata(
|
||||
pool_seqlens,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
if build_schedule_metadata
|
||||
else None
|
||||
)
|
||||
return dataclasses.replace(
|
||||
metadata,
|
||||
pooled_index_kpool=pool_size,
|
||||
pooled_index_kpool=kpool,
|
||||
pooled_cache_seqlens_int32=pool_seqlens,
|
||||
pooled_real_page_table=pooled_page_table,
|
||||
pooled_paged_mqa_schedule_metadata=schedule,
|
||||
@@ -624,13 +622,13 @@ def update_pooled_paged_mqa_metadata(
|
||||
seqlens_32: torch.Tensor,
|
||||
forward_mode: ForwardMode,
|
||||
*,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
slots_per_page: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
index_page_size: int,
|
||||
build_schedule_metadata: bool = True,
|
||||
) -> None:
|
||||
if (
|
||||
not _is_kpool_layout_enabled(pool_size, real_page_size)
|
||||
not _is_kpool_layout_enabled(kpool, physical_page_size)
|
||||
or not is_cuda()
|
||||
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
|
||||
):
|
||||
@@ -638,25 +636,23 @@ def update_pooled_paged_mqa_metadata(
|
||||
|
||||
if forward_mode.is_target_verify():
|
||||
torch.floor_divide(
|
||||
metadata.real_page_table[:, ::pool_size],
|
||||
pool_size,
|
||||
metadata.real_page_table[:, ::kpool],
|
||||
kpool,
|
||||
out=metadata.pooled_real_page_table,
|
||||
)
|
||||
return
|
||||
|
||||
if (
|
||||
metadata.pooled_index_kpool != pool_size
|
||||
metadata.pooled_index_kpool != kpool
|
||||
or metadata.pooled_cache_seqlens_int32 is None
|
||||
or metadata.pooled_real_page_table is None
|
||||
):
|
||||
return
|
||||
|
||||
pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to(
|
||||
torch.int32
|
||||
)
|
||||
pool_seqlens = torch.div(seqlens_32, kpool, rounding_mode="floor").to(torch.int32)
|
||||
metadata.pooled_cache_seqlens_int32[: pool_seqlens.shape[0]].copy_(pool_seqlens)
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
metadata.real_page_table, pool_size
|
||||
metadata.real_page_table, kpool
|
||||
).contiguous()
|
||||
metadata.pooled_real_page_table[
|
||||
: pooled_page_table.shape[0], : pooled_page_table.shape[1]
|
||||
@@ -668,7 +664,7 @@ def update_pooled_paged_mqa_metadata(
|
||||
):
|
||||
new_schedule = _compute_pool_schedule_metadata(
|
||||
metadata.pooled_cache_seqlens_int32,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
if new_schedule is not None:
|
||||
metadata.pooled_paged_mqa_schedule_metadata.copy_(new_schedule)
|
||||
@@ -678,12 +674,12 @@ def _alloc_kpool_write_plan_buffers(
|
||||
*,
|
||||
max_bs: int,
|
||||
num_draft_tokens: int,
|
||||
pool_size: int,
|
||||
kpool: int,
|
||||
device: torch.device,
|
||||
is_verify: bool,
|
||||
is_v2: bool = False,
|
||||
) -> KPoolWritePlan:
|
||||
max_closed_pools = kpool_max_closed_pools(num_draft_tokens, pool_size)
|
||||
max_closed_pools = kpool_max_closed_pools(num_draft_tokens, kpool)
|
||||
verify_extras = {}
|
||||
if is_verify:
|
||||
n_rows = max_bs * num_draft_tokens
|
||||
@@ -711,22 +707,22 @@ def init_kpool_write_plan_capture(
|
||||
metadata: DSAMetadata,
|
||||
*,
|
||||
max_bs: int,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
num_draft_tokens: int,
|
||||
device: torch.device,
|
||||
is_verify: bool,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
is_v2: bool = False,
|
||||
build_schedule_metadata: bool = True,
|
||||
) -> DSAMetadata:
|
||||
if not _is_kpool_layout_enabled(pool_size, real_page_size) or num_draft_tokens == 0:
|
||||
if not _is_kpool_layout_enabled(kpool, physical_page_size) or num_draft_tokens == 0:
|
||||
return metadata
|
||||
|
||||
plan = _alloc_kpool_write_plan_buffers(
|
||||
max_bs=max_bs,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
device=device,
|
||||
is_verify=is_verify,
|
||||
is_v2=is_v2,
|
||||
@@ -734,7 +730,7 @@ def init_kpool_write_plan_capture(
|
||||
if is_verify and build_schedule_metadata:
|
||||
schedule = _compute_pool_schedule_metadata(
|
||||
plan.pool_seqlens_per_q,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
plan = dataclasses.replace(plan, pool_schedule_metadata=schedule)
|
||||
return dataclasses.replace(metadata, kpool_write_plan=plan)
|
||||
@@ -746,14 +742,14 @@ def update_kpool_write_plan(
|
||||
write_start: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
real_page_table: torch.Tensor,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
num_draft_tokens: int,
|
||||
forward_mode: ForwardMode,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
effective_n_per_batch: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
if not _is_kpool_layout_enabled(pool_size, real_page_size) or not (
|
||||
if not _is_kpool_layout_enabled(kpool, physical_page_size) or not (
|
||||
is_cuda() or is_hip()
|
||||
):
|
||||
return
|
||||
@@ -775,9 +771,9 @@ def update_kpool_write_plan(
|
||||
write_loc_out=plan.write_loc,
|
||||
pool_seqlens_per_q_out=plan.pool_seqlens_per_q,
|
||||
seqlens_per_q_out=plan.seqlens_per_q,
|
||||
pool_size=pool_size,
|
||||
kpool=kpool,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -792,7 +788,7 @@ def update_kpool_write_plan(
|
||||
if plan.pool_schedule_metadata is not None:
|
||||
new_schedule = _compute_pool_schedule_metadata(
|
||||
plan.pool_seqlens_per_q,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
if new_schedule is not None:
|
||||
plan.pool_schedule_metadata.copy_(new_schedule)
|
||||
@@ -802,12 +798,12 @@ def init_kpool_write_plan(
|
||||
metadata: DSAMetadata,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
pool_size: int,
|
||||
real_page_size: int,
|
||||
kpool: int,
|
||||
physical_page_size: int,
|
||||
real_page_table: torch.Tensor,
|
||||
num_draft_tokens: int,
|
||||
write_start: torch.Tensor,
|
||||
slots_per_page: int,
|
||||
index_page_size: int,
|
||||
effective_n_per_batch: Optional[torch.Tensor] = None,
|
||||
build_schedule_metadata: bool = True,
|
||||
) -> DSAMetadata:
|
||||
@@ -816,7 +812,7 @@ def init_kpool_write_plan(
|
||||
is_decode = forward_mode.is_decode_or_idle()
|
||||
is_v2 = forward_mode.is_draft_extend_v2()
|
||||
is_ring_write = is_verify or is_decode or is_v2
|
||||
if not _is_kpool_layout_enabled(pool_size, real_page_size) or not is_ring_write:
|
||||
if not _is_kpool_layout_enabled(kpool, physical_page_size) or not is_ring_write:
|
||||
return metadata
|
||||
|
||||
pool = getattr(forward_batch, "token_to_kv_pool", None)
|
||||
@@ -829,12 +825,12 @@ def init_kpool_write_plan(
|
||||
metadata = init_kpool_write_plan_capture(
|
||||
metadata,
|
||||
max_bs=forward_batch.seq_lens.shape[0],
|
||||
pool_size=pool_size,
|
||||
real_page_size=real_page_size,
|
||||
kpool=kpool,
|
||||
physical_page_size=physical_page_size,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
device=forward_batch.seq_lens.device,
|
||||
is_verify=is_verify or is_v2,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
is_v2=is_v2,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
@@ -843,11 +839,11 @@ def init_kpool_write_plan(
|
||||
write_start=write_start,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
real_page_table=real_page_table,
|
||||
pool_size=pool_size,
|
||||
real_page_size=real_page_size,
|
||||
kpool=kpool,
|
||||
physical_page_size=physical_page_size,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
forward_mode=forward_mode,
|
||||
slots_per_page=slots_per_page,
|
||||
index_page_size=index_page_size,
|
||||
effective_n_per_batch=effective_n_per_batch,
|
||||
)
|
||||
return metadata
|
||||
|
||||
@@ -446,7 +446,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.real_page_size = model_runner.page_size // self.dsa_index_kpool
|
||||
self.physical_page_size = model_runner.page_size // self.dsa_index_kpool
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self._memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
@@ -938,7 +938,7 @@ class DeepseekSparseAttnBackend(
|
||||
return self.req_to_token.shape[1]
|
||||
|
||||
def _transform_table_1_to_real(self, page_table: torch.Tensor) -> torch.Tensor:
|
||||
page_size = self.real_page_size
|
||||
page_size = self.physical_page_size
|
||||
if page_size == 1:
|
||||
return page_table
|
||||
max_seqlen_k = page_table.shape[1]
|
||||
@@ -1021,11 +1021,11 @@ class DeepseekSparseAttnBackend(
|
||||
use_kpool = self.dsa_index_kpool > 1
|
||||
if use_kpool:
|
||||
assert (
|
||||
self.real_page_size == 64
|
||||
and self.real_page_size % self.dsa_index_kpool == 0
|
||||
self.physical_page_size == 64
|
||||
and self.physical_page_size % self.dsa_index_kpool == 0
|
||||
), (
|
||||
f"kpool path requires page_size == 64 and page_size % pool_size == 0; "
|
||||
f"got page_size={self.real_page_size}, pool_size={self.dsa_index_kpool}."
|
||||
f"got page_size={self.physical_page_size}, pool_size={self.dsa_index_kpool}."
|
||||
)
|
||||
kpool_inputs = _KPoolForwardInputs()
|
||||
|
||||
@@ -1239,7 +1239,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
|
||||
metadata = DSAMetadata(
|
||||
page_size=self.real_page_size,
|
||||
page_size=self.physical_page_size,
|
||||
cache_seqlens_int32=cache_seqlens_int32,
|
||||
max_seq_len_q=max_seqlen_q,
|
||||
max_seq_len_k=max_seqlen_k,
|
||||
@@ -1382,7 +1382,7 @@ class DeepseekSparseAttnBackend(
|
||||
self.dsa_drop_wide_page_table = (
|
||||
is_cuda()
|
||||
and not _is_hip
|
||||
and self.real_page_size > 1
|
||||
and self.physical_page_size > 1
|
||||
and self.hisparse_coordinator is None
|
||||
and not self.speculative_num_draft_tokens
|
||||
# kpool's PAGED fused-topk mapping still reads page_table_1.
|
||||
@@ -1435,7 +1435,8 @@ class DeepseekSparseAttnBackend(
|
||||
"real_page_table": (
|
||||
torch.zeros(
|
||||
max_num_tokens,
|
||||
(max_ctx_len + self.real_page_size - 1) // self.real_page_size,
|
||||
(max_ctx_len + self.physical_page_size - 1)
|
||||
// self.physical_page_size,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
@@ -1627,7 +1628,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
|
||||
metadata = DSAMetadata(
|
||||
page_size=self.real_page_size,
|
||||
page_size=self.physical_page_size,
|
||||
cache_seqlens_int32=cache_seqlens_int32,
|
||||
max_seq_len_q=max_seqlen_q,
|
||||
max_seq_len_k=max_seqlen_k,
|
||||
@@ -1711,7 +1712,7 @@ class DeepseekSparseAttnBackend(
|
||||
bs=bs,
|
||||
max_len=max_len,
|
||||
dsa_index_topk=self.dsa_index_topk,
|
||||
real_page_size=self.real_page_size,
|
||||
physical_page_size=self.physical_page_size,
|
||||
)
|
||||
cache_seqlens = metadata.cache_seqlens_int32
|
||||
dsa_cache_seqlens = metadata.dsa_cache_seqlens_int32
|
||||
@@ -1766,7 +1767,7 @@ class DeepseekSparseAttnBackend(
|
||||
bs=bs,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
dsa_index_topk=self.dsa_index_topk,
|
||||
real_page_size=self.real_page_size,
|
||||
physical_page_size=self.physical_page_size,
|
||||
next_n=self.speculative_num_draft_tokens,
|
||||
paged_mqa_ctx_lens_2d=paged_mqa_ctx_lens_2d,
|
||||
)
|
||||
@@ -1854,7 +1855,7 @@ class DeepseekSparseAttnBackend(
|
||||
total_len=total_extend_len,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
dsa_index_topk=self.dsa_index_topk,
|
||||
real_page_size=self.real_page_size,
|
||||
physical_page_size=self.physical_page_size,
|
||||
max_extend_len=self.speculative_num_draft_tokens,
|
||||
max_total_len=bs * self.speculative_num_draft_tokens,
|
||||
static_extend_len=True,
|
||||
@@ -1934,8 +1935,8 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
# NOTE(dark): (dsa-) cu_seqlens_q is always arange, no need to copy
|
||||
|
||||
assert self.real_page_size == metadata.page_size
|
||||
if self.real_page_size > 1:
|
||||
assert self.physical_page_size == metadata.page_size
|
||||
if self.physical_page_size > 1:
|
||||
if not used_fused_metadata_generation:
|
||||
real_table = self._transform_table_1_to_real(page_indices)
|
||||
new_rows = real_table.shape[0]
|
||||
@@ -3052,7 +3053,7 @@ class DeepseekSparseAttnBackend(
|
||||
indices=page_table_1,
|
||||
seq_lens=seq_lens,
|
||||
workspace_buffer=self.workspace_buffer,
|
||||
page_size=self.real_page_size,
|
||||
page_size=self.physical_page_size,
|
||||
kv_cache_dim=self.kv_cache_dim,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
@@ -3089,8 +3090,8 @@ class DeepseekSparseAttnBackend(
|
||||
else:
|
||||
q_input = q_all
|
||||
|
||||
kv_cache = kv_cache.view(-1, self.real_page_size, 1, self.kv_cache_dim)
|
||||
assert self.real_page_size == 64, "only page size 64 is supported"
|
||||
kv_cache = kv_cache.view(-1, self.physical_page_size, 1, self.kv_cache_dim)
|
||||
assert self.physical_page_size == 64, "only page size 64 is supported"
|
||||
|
||||
if not self.dsa_kv_cache_store_fp8:
|
||||
# inefficiently quantize the whole cache
|
||||
@@ -3331,7 +3332,7 @@ class DeepseekSparseAttnBackend(
|
||||
|
||||
D_ckv = kv_cache.shape[-1]
|
||||
# kv_cache: (N_tokens, 1, D_ckv) from MLATokenToKVPool → (N_pages, page_size, D_ckv)
|
||||
kv_paged = kv_cache.view(-1, self.real_page_size, D_ckv)
|
||||
kv_paged = kv_cache.view(-1, self.physical_page_size, D_ckv)
|
||||
|
||||
block_table = metadata.real_page_table # (B, max_pages), page-indexed int32
|
||||
seq_lens_k = metadata.cache_seqlens_int32 # (B,) total KV lengths
|
||||
@@ -3607,7 +3608,9 @@ class DeepseekSparseAttnBackend(
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(layer, cache_loc, k, k_rope)
|
||||
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = k_cache.view(-1, self.real_page_size, self.kv_cache_dim).unsqueeze(1)
|
||||
kv_cache = k_cache.view(
|
||||
-1, self.physical_page_size, self.kv_cache_dim
|
||||
).unsqueeze(1)
|
||||
|
||||
if merge_query:
|
||||
q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
|
||||
@@ -3677,7 +3680,7 @@ class DeepseekSparseAttnBackend(
|
||||
multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size)
|
||||
|
||||
q = q_all.view(batch_size, 1, num_heads, head_dim)
|
||||
kv = kv_cache.view(-1, 1, self.real_page_size, self.kv_cache_dim)
|
||||
kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim)
|
||||
block_tables = page_table_1.unsqueeze(1)
|
||||
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ class IndexKeyCache:
|
||||
pool = self.pool
|
||||
return (
|
||||
num_pages,
|
||||
pool.slots_per_page
|
||||
pool.index_page_size
|
||||
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
|
||||
)
|
||||
|
||||
|
||||
@@ -4290,8 +4290,8 @@ class HybridLinearKVPool(KVCache):
|
||||
return getattr(self.full_kv_pool, "tail_extra_slots", 0)
|
||||
|
||||
@property
|
||||
def slots_per_page(self) -> int:
|
||||
return getattr(self.full_kv_pool, "slots_per_page", self.page_size)
|
||||
def index_page_size(self) -> int:
|
||||
return getattr(self.full_kv_pool, "index_page_size", self.page_size)
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
return self.full_kv_pool.get_kv_size_bytes()
|
||||
@@ -5153,7 +5153,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
assert self.page_size % index_kpool == 0, (
|
||||
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
|
||||
)
|
||||
self.slots_per_page = self.page_size // index_kpool
|
||||
self.index_page_size = self.page_size // index_kpool
|
||||
if index_buf_size is None:
|
||||
index_buf_size = size
|
||||
self.index_buf_size = index_buf_size
|
||||
|
||||
@@ -98,7 +98,7 @@ def make_dsa_indexer_pool_decl(
|
||||
pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER
|
||||
) -> HostPoolDecl:
|
||||
"""Index key buffers riding on the full-KV pages: indices and layout both follow KV."""
|
||||
index_page_bytes = pool.slots_per_page * dsa_indexer_bytes_per_token_per_layer(
|
||||
index_page_bytes = pool.index_page_size * dsa_indexer_bytes_per_token_per_layer(
|
||||
index_head_dim=pool.index_head_dim,
|
||||
quant_block_size=pool.quant_block_size,
|
||||
)
|
||||
|
||||
@@ -32,7 +32,7 @@ def make_backend(mode, seq, req, *, fusion=True):
|
||||
backend.device = torch.device("cuda")
|
||||
backend.device_sm_major = torch.cuda.get_device_capability()[0]
|
||||
backend.num_q_heads = 64
|
||||
backend.real_page_size = 64
|
||||
backend.physical_page_size = 64
|
||||
backend.dsa_index_topk = TOPK
|
||||
backend.dsa_index_kpool = POOL
|
||||
backend.speculative_num_draft_tokens = NEXT_N
|
||||
@@ -40,7 +40,7 @@ def make_backend(mode, seq, req, *, fusion=True):
|
||||
backend.dsa_decode_impl = "fa3"
|
||||
backend.dsa_prefill_impl = "fa3"
|
||||
backend.enable_auto_select_prefill_impl = False
|
||||
backend.token_to_kv_pool = SimpleNamespace(slots_per_page=64)
|
||||
backend.token_to_kv_pool = SimpleNamespace(index_page_size=64)
|
||||
# Only attention-dispatch state is synthetic; every metadata kernel is real.
|
||||
backend._is_in_breakable_cuda_graph = lambda: False
|
||||
backend._is_in_tc_piecewise_cuda_graph = lambda: False
|
||||
|
||||
Reference in New Issue
Block a user