[MemCache] Skip SWA request-cap pool sizing when chunked prefill is off (#43203)
This commit is contained in:
@@ -948,7 +948,12 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
return False
|
||||
if not get_memory().disable_radix_cache:
|
||||
return False
|
||||
if get_schedule().chunked_prefill_size is None:
|
||||
# The cap holds at most one prefill chunk per batch in flight; an
|
||||
# unchunked prefill writes the whole prompt into the SWA pool.
|
||||
chunked_prefill_size = get_schedule().chunked_prefill_size
|
||||
if get_disagg().disaggregation_mode != "decode" and (
|
||||
chunked_prefill_size is None or chunked_prefill_size <= 0
|
||||
):
|
||||
return False
|
||||
if kvc.sliding_window_size is None:
|
||||
return False
|
||||
|
||||
@@ -1004,7 +1004,7 @@ class TestFactory(CustomTestCase):
|
||||
|
||||
def test_chunk_cap_configurator_selection(self):
|
||||
# SWAChunkCapPoolConfigurator is selected only when max_running_requests is set.
|
||||
def _cfg(max_running_requests):
|
||||
def _cfg(max_running_requests, chunked_prefill_size=4):
|
||||
mr = _make_model_runner(
|
||||
self,
|
||||
is_hybrid_swa=True,
|
||||
@@ -1012,7 +1012,7 @@ class TestFactory(CustomTestCase):
|
||||
swa_attention_layer_ids=[1],
|
||||
swa_num_kv_heads=4,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=4,
|
||||
chunked_prefill_size=chunked_prefill_size,
|
||||
sliding_window_size=8,
|
||||
max_running_requests=max_running_requests,
|
||||
)
|
||||
@@ -1029,6 +1029,10 @@ class TestFactory(CustomTestCase):
|
||||
|
||||
self.assertIsInstance(_cfg(2), SWAChunkCapPoolConfigurator)
|
||||
self.assertNotIsInstance(_cfg(None), SWAChunkCapPoolConfigurator)
|
||||
# Chunked prefill off: a whole prompt can exceed the cap.
|
||||
self.assertNotIsInstance(
|
||||
_cfg(2, chunked_prefill_size=-1), SWAChunkCapPoolConfigurator
|
||||
)
|
||||
|
||||
|
||||
class TestDflashDraftKvBudget(CustomTestCase):
|
||||
|
||||
Reference in New Issue
Block a user