[Perf] [NPU] mem_cache: Increase GLM-5.2-W4A8C8 KV cache capacity with compact DSA indexer allocation (#44436)

This commit is contained in:
Jensen
2026-10-20 16:30:01 +03:00
committed by GitHub
parent aa91108f64
commit 3bf4cbc6cf
4 changed files with 129 additions and 8 deletions
@@ -2060,10 +2060,10 @@ class KVCacheConfigurator:
else:
index_size = max_total_num_tokens * dcp_size
is_arch35 = is_npu_arch35()
use_compact_indexer_layout = (
is_dsa_model
and is_arch35
and _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker)
# Shared layers never access the indexer; pool sizing uses the same gate
# so the saved indexer memory can be used for additional KV tokens.
use_compact_indexer_layout = is_dsa_model and _should_elide_dsa_index_k(
is_draft_worker=self.is_draft_worker
)
indexer_layer_ids = None
if use_compact_indexer_layout:
@@ -548,15 +548,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
DSATokenToKVPool.index_k_with_scale_buffer_dtype
)
if _is_npu:
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
dtype = kvc.kv_cache_dtype
# GPU sizing above assumes FP8 indexers; NPU also needs BF16 sizing.
if dtype != torch.float8_e4m3fn:
indexer_size_per_token = index_head_dim
element_size = torch._utils._element_size(dtype)
if not is_npu_arch35():
allocate_all_layers = True
memory_config = get_memory()
indexer_ratio = 1
if memory_config.enable_hisparse: