[Fix] scheduler: stop latching batch_is_full when nothing can run (#41872)
Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
hnyls2002
Liangsheng Yin
parent
bf703013f2
commit
a297cd3da1
@@ -3942,6 +3942,10 @@ class Scheduler(
|
||||
if self.enable_priority_preemption or self.is_hybrid_swa:
|
||||
# Reset batch_is_full to try preemption with a prefill adder.
|
||||
running_batch.batch_is_full = False
|
||||
if running_batch.is_empty():
|
||||
# An empty batch never reaches update_running_batch, the only
|
||||
# other place that clears the flag, so admission would stay shut.
|
||||
running_batch.batch_is_full = False
|
||||
|
||||
if (
|
||||
running_batch.batch_is_full or len(self.waiting_queue) == 0
|
||||
@@ -4127,17 +4131,10 @@ class Scheduler(
|
||||
|
||||
if res != AddReqResult.CONTINUE:
|
||||
if res == AddReqResult.NO_TOKEN:
|
||||
if (
|
||||
self.enable_hierarchical_cache
|
||||
or self.enable_lmcache
|
||||
or self.enable_unified_cache_external_linker
|
||||
):
|
||||
# Set batch_is_full after making sure there are requests that can be served
|
||||
running_batch.batch_is_full = len(adder.can_run_list) > 0 or (
|
||||
not running_batch.is_empty()
|
||||
)
|
||||
else:
|
||||
running_batch.batch_is_full = True
|
||||
# Set batch_is_full after making sure there are requests that can be served
|
||||
running_batch.batch_is_full = len(adder.can_run_list) > 0 or (
|
||||
not running_batch.is_empty()
|
||||
)
|
||||
# revert matched mamba idx to avoid memory leak, if req is not added.
|
||||
# Only free if the slot was freshly allocated in this batch (not
|
||||
# pre-existing from a session). Session-held slots have their own
|
||||
|
||||
Reference in New Issue
Block a user