From 3bf4cbc6cf18b539d88c08c17cb97d82ec82c951 Mon Sep 17 00:00:00 2001 From: Jensen Date: Tue, 20 Oct 2026 21:30:01 +0800 Subject: [PATCH] [Perf] [NPU] mem_cache: Increase GLM-5.2-W4A8C8 KV cache capacity with compact DSA indexer allocation (#44436) --- .../srt/mem_cache/kv_cache_configurator.py | 8 +- .../srt/model_executor/pool_configurator.py | 4 - .../test_kv_cache_configurator_npu_indexer.py | 82 +++++++++++++++++++ .../model_executor/test_pool_configurator.py | 43 ++++++++++ 4 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_kv_cache_configurator_npu_indexer.py diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index b8801be1cd..c4e87f316e 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -2060,10 +2060,10 @@ class KVCacheConfigurator: else: index_size = max_total_num_tokens * dcp_size is_arch35 = is_npu_arch35() - use_compact_indexer_layout = ( - is_dsa_model - and is_arch35 - and _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker) + # Shared layers never access the indexer; pool sizing uses the same gate + # so the saved indexer memory can be used for additional KV tokens. + use_compact_indexer_layout = is_dsa_model and _should_elide_dsa_index_k( + is_draft_worker=self.is_draft_worker ) indexer_layer_ids = None if use_compact_indexer_layout: diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 63782a3dd1..614d914a95 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -548,15 +548,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): DSATokenToKVPool.index_k_with_scale_buffer_dtype ) if _is_npu: - from sglang.srt.hardware_backend.npu.utils import is_npu_arch35 - dtype = kvc.kv_cache_dtype # GPU sizing above assumes FP8 indexers; NPU also needs BF16 sizing. if dtype != torch.float8_e4m3fn: indexer_size_per_token = index_head_dim element_size = torch._utils._element_size(dtype) - if not is_npu_arch35(): - allocate_all_layers = True memory_config = get_memory() indexer_ratio = 1 if memory_config.enable_hisparse: diff --git a/test/registered/unit/mem_cache/test_kv_cache_configurator_npu_indexer.py b/test/registered/unit/mem_cache/test_kv_cache_configurator_npu_indexer.py new file mode 100644 index 0000000000..32a9faeb78 --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_cache_configurator_npu_indexer.py @@ -0,0 +1,82 @@ +"""CPU regressions for NPU DSA indexer pool layer ownership.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.mem_cache import kv_cache_configurator as kcc +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, stage="base-a", runner_config="cpu") + + +class TestNPUIndexerLayerLayout(CustomTestCase): + def test_non_arch35_dsa_pool_uses_only_active_indexer_layers(self): + """Shared layers must not consume indexer slots on non-arch35 NPU.""" + configurator = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + configurator.device = "cpu" + configurator.is_draft_worker = False + configurator.kv_cache_dtype = torch.bfloat16 + configurator.layer_info = SimpleNamespace( + start_layer=10, + end_layer=14, + num_effective_layers=4, + ) + configurator.model_config = SimpleNamespace( + hf_config=object(), + kv_lora_rank=16, + qk_rope_head_dim=8, + index_head_dim=32, + ) + pool = object() + + with ( + patch.object( + kcc, + "get_parallel", + return_value=SimpleNamespace(attn_dcp_size=2), + ), + patch.object( + kcc, + "get_schedule", + return_value=SimpleNamespace(page_size=16), + ), + patch.object( + kcc, + "get_exec", + return_value=SimpleNamespace( + features=SimpleNamespace(enable_memory_saver=False) + ), + ), + patch.object(kcc, "_should_elide_dsa_index_k", return_value=True), + patch.object( + kcc, + "dsa_layer_skips_topk", + side_effect=lambda _config, layer_id: layer_id in (11, 13), + ), + patch( + "sglang.srt.hardware_backend.npu.utils.is_npu_arch35", + return_value=False, + ), + patch( + "sglang.srt.hardware_backend.npu.memory_pool_npu.NPUMLATokenToKVPool", + return_value=pool, + ) as pool_class, + ): + result = configurator._build_ascend_mla_kv_pool( + max_total_num_tokens=64, + is_dsa_model=True, + ) + + self.assertIs(result, pool) + self.assertEqual(pool_class.call_args.args, (64,)) + self.assertEqual(pool_class.call_args.kwargs["index_size"], 128) + self.assertEqual(pool_class.call_args.kwargs["indexer_layer_ids"], (10, 12)) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index b534baf8c4..1aee4ddca0 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1084,6 +1084,49 @@ class TestEagleConfigurator(CustomTestCase): class TestDSAIndexerAllocationPolicy(CustomTestCase): + @patch( + "sglang.srt.mem_cache.kv_cache_configurator.calculate_mla_kv_cache_dim", + return_value=576, + ) + def test_non_arch35_npu_budget_uses_compact_indexer_capacity( + self, + _mock_calculate_mla_kv_cache_dim, + ): + num_layers = 6 + mr = _make_model_runner(self, num_layers=num_layers, use_mla_backend=True) + _configure_dsa_model(mr) + mr.model_config.hf_config.indexer_types = [ + "full", + "shared", + "full", + "shared", + "shared", + "shared", + ] + available_bytes = 1_000_000 + + with ( + mock_cpu_env(kv_size=2), + patch("sglang.srt.model_executor.pool_configurator._is_npu", True), + ): + from sglang.srt.model_executor.pool_configurator import ( + DefaultPoolConfigurator, + ) + + configurator = DefaultPoolConfigurator(mr) + pool_sizes = configurator.calculate_pool_sizes( + available_bytes, + page_size=1, + ) + + compact_cell_size = 576 * num_layers * 2 + 128 * 2 * 2 + dense_cell_size = (576 + 128) * num_layers * 2 + self.assertEqual(configurator._cell_size, compact_cell_size) + self.assertGreater( + pool_sizes.max_total_num_tokens, + available_bytes // dense_cell_size, + ) + @patch( "sglang.srt.mem_cache.kv_cache_configurator.calculate_mla_kv_cache_dim", return_value=576,