[Feat] KVShard: Add attention and indexer support for page-sharded DSA caches (#44280)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Shunkangz
2026-10-23 00:16:35 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 8cd3c494a3
commit cc811ff0c3
6 changed files with 1250 additions and 578 deletions
@@ -152,6 +152,7 @@ if _use_aiter:
from aiter.ops.cache import indexer_k_quant_and_cache
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.attention.kv_shard_hooks import get_kv_shard_pool
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.linear import ReplicatedLinear
@@ -240,6 +241,14 @@ def _make_eager_idle_topk_result(
)
def _index_k_write_targets(pool, layer_id: int, loc: torch.Tensor):
"""(index-K buffer, token slots, selected token rows or None) to write."""
shard_pool = get_kv_shard_pool(pool)
if shard_pool is None:
return ((pool.get_index_k_with_scale_buffer(layer_id=layer_id), loc, None),)
return shard_pool.index_key_cache.write_targets(layer_id, loc)
def rotate_activation(x: torch.Tensor) -> torch.Tensor:
if _is_hip:
from fast_hadamard_transform import hadamard_transform
@@ -1026,17 +1035,18 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
and out_cache_loc is not None
and can_use_dsa_fused_store(torch.bfloat16, out_cache_loc.dtype, page_size)
):
fused_k_indexer_norm_rope_store(
key_raw,
pool.get_index_k_with_scale_buffer(layer_id=layer_id),
out_cache_loc,
self.k_norm.weight,
self.k_norm.bias,
self.k_norm.variance_epsilon,
self._indexer_cos_sin_cache,
positions,
page_size,
)
for buf, loc, rows in _index_k_write_targets(pool, layer_id, out_cache_loc):
fused_k_indexer_norm_rope_store(
key_raw if rows is None else key_raw.index_select(0, rows),
buf,
loc,
self.k_norm.weight,
self.k_norm.bias,
self.k_norm.variance_epsilon,
self._indexer_cos_sin_cache,
positions if rows is None else positions.index_select(0, rows),
page_size,
)
return
# Fallback: separate K kernel + store kernel.
@@ -1510,7 +1520,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
if _is_hip and not _use_aiter_preshuffle:
block_tables = metadata.get_page_table_1()
else:
block_tables = metadata.get_page_table_64()
block_tables = metadata.get_logical_page_table()
assert (
forward_batch.seq_lens_cpu is not None
@@ -1830,13 +1840,13 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
)
):
# NOTE: wrapper already normalizes shape/contiguity and asserts dtypes.
buf = pool.get_index_k_with_scale_buffer(layer_id=layer_id)
fused_store_index_k_cache(
key,
buf,
out_cache_loc,
pool.index_page_size,
)
for buf, loc, rows in _index_k_write_targets(pool, layer_id, out_cache_loc):
fused_store_index_k_cache(
key if rows is None else key.index_select(0, rows),
buf,
loc,
pool.index_page_size,
)
return
# Fast path: AITER fused quant + cache store
@@ -32,6 +32,10 @@ class BaseIndexerMetadata(ABC):
The page size of the table is 64.
"""
def get_logical_page_table(self) -> torch.Tensor:
"""Page IDs for pool-managed reads, before any pool address translation."""
return self.get_page_table_64()
@abstractmethod
def get_page_table_1(self) -> torch.Tensor:
"""
@@ -105,6 +109,10 @@ class DSAIndexerMetadata(BaseIndexerMetadata):
def get_page_table_64(self) -> torch.Tensor:
return self.attn_metadata.real_page_table
def get_logical_page_table(self) -> torch.Tensor:
logical = self.attn_metadata.indexer_logical_page_table
return self.get_page_table_64() if logical is None else logical
def get_page_table_1(self) -> torch.Tensor:
return self.attn_metadata.page_table_1
@@ -87,6 +87,10 @@ from sglang.srt.layers.attention.dsa.utils import (
resolve_paged_mqa_logits_metadata_fn,
should_use_dsa_fused_topk,
)
from sglang.srt.layers.attention.kv_shard_hooks import (
get_kv_shard_pool,
prepare_kv_shard_forward,
)
from sglang.srt.layers.attention.trtllm_mla_backend import (
grow_multi_ctas_kv_counter_buffer_if_needed,
make_persistent_multi_ctas_kv_counter_buffer,
@@ -295,6 +299,9 @@ class DSAMetadata:
# k_start and k_end in kv cache for each token.
indexer_k_start_end: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
# Pool-managed index reads take logical page IDs and translate them once.
# Raw attention/indexer kernels instead use the scratch-addressed tables.
indexer_logical_page_table: Optional[torch.Tensor] = None
# seq lens for each batch.
indexer_seq_lens_cpu: Optional[torch.Tensor] = None
# seq lens for each batch.
@@ -551,11 +558,16 @@ class DeepseekSparseAttnBackend(
assert model_runner.req_to_token_pool is not None
self.req_to_token_pool = model_runner.req_to_token_pool
self.token_to_kv_pool = model_runner.token_to_kv_pool
self._kv_shard_pool = get_kv_shard_pool(self.token_to_kv_pool)
self.hisparse_coordinator = model_runner.hisparse_coordinator
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.use_mha: bool = False
self.supports_mha_one_shot: bool = self.dcp_size == 1
# Dense MHA expands prefix KV outside the DSA metadata path. Keep
# sharded prefill on the sparse kernels with explicit scratch addresses.
self.supports_mha_one_shot: bool = (
self.dcp_size == 1 and self._kv_shard_pool is None
)
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
if (
@@ -1106,6 +1118,11 @@ class DeepseekSparseAttnBackend(
forward_batch.req_pool_indices, :max_seqlen_k
]
kv_shard_extend = self._kv_shard_pool is not None and prepare_kv_shard_forward(
self._kv_shard_pool, self.req_to_token, forward_batch
)
indexer_logical_page_table = None
page_table_1_flattened = None
topk_indices_offset = None
@@ -1264,6 +1281,14 @@ class DeepseekSparseAttnBackend(
)
page_table = page_table[bs_idx, :max_seqlen_k]
if kv_shard_extend:
# Kernels read the gathered prefix/chunk scratch; the indexer
# pool read takes logical pages and translates them itself.
indexer_logical_page_table = self._transform_table_1_to_real(page_table)
page_table = self._kv_shard_pool.translate_loc_to_scratch(
page_table
).to(page_table.dtype)
if any(forward_batch.extend_prefix_lens_cpu) or bs_idx_cpu is not None:
max_seqlen_q = (
max(extend_seq_lens_cpu) if len(extend_seq_lens_cpu) != 0 else 1
@@ -1302,7 +1327,11 @@ class DeepseekSparseAttnBackend(
kv_cache_capacity = (
self.token_to_kv_pool.size + self.token_to_kv_pool.page_size
)
if forward_batch.seq_lens_sum > kv_cache_capacity:
if (
# Sharded extends index scratch rows, bounded by construction.
not kv_shard_extend
and forward_batch.seq_lens_sum > kv_cache_capacity
):
max_idx = page_table_1_flattened.max().item()
assert max_idx < kv_cache_capacity, (
f"Invalid page table index: max={max_idx}, "
@@ -1381,6 +1410,7 @@ class DeepseekSparseAttnBackend(
dsa_max_seqlen_q=1,
topk_indices_offset=topk_indices_offset,
indexer_k_start_end=indexer_k_start_end,
indexer_logical_page_table=indexer_logical_page_table,
indexer_seq_lens_cpu=indexer_seq_lens_cpu,
indexer_seq_lens=indexer_seq_lens,
token_to_batch_idx=token_to_batch_idx,
@@ -780,6 +780,40 @@ class _PageInterleaveIndexKeyCache(IndexKeyCache):
max_seq_len,
)
def write_targets(
self, layer_id: int, loc: torch.Tensor
) -> List[Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]]:
"""(buffer, token slots, selected token rows or None) for a write.
During an extend every token is staged at its scratch slot; owned
tokens also persist at their local slots. Writers apply their own
quantization to each target, so sharded bytes match unsharded ones.
"""
if loc.numel() == 0:
return []
pool = self.pool
local = self.buffer[layer_id - pool.start_layer]
assert local.shape[0], "cannot write indexer keys for a skip-topk layer"
targets = []
if pool._shard_extend_active:
targets.append(
(
pool._slots[layer_id % 2].tensors["index_k"],
pool.translate_loc_to_scratch(loc),
None,
)
)
# Indexer callers may reuse or mutate their location tensors. Compute
# fresh indices instead of memoizing only their pointer and length.
owned_idx = torch.nonzero(
pool.placement.local_mask(loc, pool.shard_rank)
).squeeze(1)
if owned_idx.numel():
targets.append(
(local, pool.placement.local_index(loc[owned_idx]), owned_idx)
)
return targets
def store_quantized(
self,
layer_id: int,
@@ -787,32 +821,17 @@ class _PageInterleaveIndexKeyCache(IndexKeyCache):
index_k: torch.Tensor,
index_k_scale: torch.Tensor,
) -> None:
if loc.numel() == 0:
return
pool = self.pool
assert self.buffer[layer_id - pool.start_layer].shape[0], (
"cannot write indexer keys for a skip-topk layer"
)
if pool._shard_extend_active:
for buf, slots, rows in self.write_targets(layer_id, loc):
dsa_index_k_cache.SetKAndS.execute(
pool=pool,
buf=pool._slots[layer_id % 2].tensors["index_k"],
loc=pool.translate_loc_to_scratch(loc),
index_k=index_k,
index_k_scale=index_k_scale,
)
# Indexer callers may reuse or mutate their location tensors. Compute
# fresh indices instead of memoizing only their pointer and length.
owned_idx = torch.nonzero(
pool.placement.local_mask(loc, pool.shard_rank)
).squeeze(1)
local_rows = pool.placement.local_index(loc[owned_idx])
if owned_idx.numel():
super().store_quantized(
layer_id,
local_rows,
index_k.index_select(0, owned_idx),
index_k_scale.index_select(0, owned_idx),
pool=self.pool,
buf=buf,
loc=slots,
index_k=index_k if rows is None else index_k.index_select(0, rows),
index_k_scale=(
index_k_scale
if rows is None
else index_k_scale.index_select(0, rows)
),
)
@@ -840,8 +859,8 @@ class PageInterleaveDSATokenToKVPool(PageInterleaveMLATokenToKVPool, DSATokenToK
expose owner-local storage; ``index_key_cache.get_local_buffer`` always
exposes that storage.
This is a pool-level implementation. DSA backend metadata, fused indexer
stores, and model-runner selection still require separate integration.
This is a pool-level implementation. Model-runner selection still
requires separate integration.
"""
def __init__(