[DSA] Skip indexer KV cache for skip-topk layers (#30531)
Co-authored-by: Brayden Zhong <brayden@radixark.ai> Co-authored-by: mmangkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Brayden Zhong
mmangkad
parent
7c423cfd41
commit
8cc112d486
@@ -4365,6 +4365,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
start_layer: Optional[int] = None,
|
||||
end_layer: Optional[int] = None,
|
||||
index_buf_size: Optional[int] = None,
|
||||
skip_topk_layers: Optional[List[bool]] = None,
|
||||
):
|
||||
override_dim = (
|
||||
kv_cache_dim if kv_cache_dim != kv_lora_rank + qk_rope_head_dim else None
|
||||
@@ -4393,6 +4394,13 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
# num head == 1 and head dim == 128 for index_k in DSA
|
||||
assert index_head_dim == 128
|
||||
|
||||
self.skip_topk_layers = (
|
||||
list(skip_topk_layers)
|
||||
if skip_topk_layers is not None
|
||||
else [False] * layer_num
|
||||
)
|
||||
assert len(self.skip_topk_layers) == layer_num
|
||||
|
||||
if _is_hip:
|
||||
if aiter_can_use_preshuffle_paged_mqa():
|
||||
assert (
|
||||
|
||||
Reference in New Issue
Block a user