[Perf] [NPU] mem_cache: Increase GLM-5.2-W4A8C8 KV cache capacity with compact DSA indexer allocation (#44436)
This commit is contained in:
@@ -2060,10 +2060,10 @@ class KVCacheConfigurator:
|
||||
else:
|
||||
index_size = max_total_num_tokens * dcp_size
|
||||
is_arch35 = is_npu_arch35()
|
||||
use_compact_indexer_layout = (
|
||||
is_dsa_model
|
||||
and is_arch35
|
||||
and _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker)
|
||||
# Shared layers never access the indexer; pool sizing uses the same gate
|
||||
# so the saved indexer memory can be used for additional KV tokens.
|
||||
use_compact_indexer_layout = is_dsa_model and _should_elide_dsa_index_k(
|
||||
is_draft_worker=self.is_draft_worker
|
||||
)
|
||||
indexer_layer_ids = None
|
||||
if use_compact_indexer_layout:
|
||||
|
||||
@@ -548,15 +548,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
DSATokenToKVPool.index_k_with_scale_buffer_dtype
|
||||
)
|
||||
if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
|
||||
|
||||
dtype = kvc.kv_cache_dtype
|
||||
# GPU sizing above assumes FP8 indexers; NPU also needs BF16 sizing.
|
||||
if dtype != torch.float8_e4m3fn:
|
||||
indexer_size_per_token = index_head_dim
|
||||
element_size = torch._utils._element_size(dtype)
|
||||
if not is_npu_arch35():
|
||||
allocate_all_layers = True
|
||||
memory_config = get_memory()
|
||||
indexer_ratio = 1
|
||||
if memory_config.enable_hisparse:
|
||||
|
||||
Reference in New Issue
Block a user