From ae4da8cd78472d6906e259ce3776f23013328686 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 8 Oct 2026 16:19:35 -0700 Subject: [PATCH] [MemCache] Skip SWA request-cap pool sizing when chunked prefill is off (#43203) --- python/sglang/srt/model_executor/pool_configurator.py | 7 ++++++- .../unit/model_executor/test_pool_configurator.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index a68cf10fe4..29301ba3a3 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -948,7 +948,12 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator): return False if not get_memory().disable_radix_cache: return False - if get_schedule().chunked_prefill_size is None: + # The cap holds at most one prefill chunk per batch in flight; an + # unchunked prefill writes the whole prompt into the SWA pool. + chunked_prefill_size = get_schedule().chunked_prefill_size + if get_disagg().disaggregation_mode != "decode" and ( + chunked_prefill_size is None or chunked_prefill_size <= 0 + ): return False if kvc.sliding_window_size is None: return False diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 5ff28606ac..73bd2a4161 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1004,7 +1004,7 @@ class TestFactory(CustomTestCase): def test_chunk_cap_configurator_selection(self): # SWAChunkCapPoolConfigurator is selected only when max_running_requests is set. - def _cfg(max_running_requests): + def _cfg(max_running_requests, chunked_prefill_size=4): mr = _make_model_runner( self, is_hybrid_swa=True, @@ -1012,7 +1012,7 @@ class TestFactory(CustomTestCase): swa_attention_layer_ids=[1], swa_num_kv_heads=4, disable_radix_cache=True, - chunked_prefill_size=4, + chunked_prefill_size=chunked_prefill_size, sliding_window_size=8, max_running_requests=max_running_requests, ) @@ -1029,6 +1029,10 @@ class TestFactory(CustomTestCase): self.assertIsInstance(_cfg(2), SWAChunkCapPoolConfigurator) self.assertNotIsInstance(_cfg(None), SWAChunkCapPoolConfigurator) + # Chunked prefill off: a whole prompt can exceed the cap. + self.assertNotIsInstance( + _cfg(2, chunked_prefill_size=-1), SWAChunkCapPoolConfigurator + ) class TestDflashDraftKvBudget(CustomTestCase):