[PD][LoRA] Gate decode admission on adapter slots (#39332)
This commit is contained in:
@@ -1188,6 +1188,19 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
- len(self.transfer_queue.queue),
|
||||
)
|
||||
|
||||
if self.scheduler.enable_lora:
|
||||
running_batches = (
|
||||
self.scheduler.running_mbs
|
||||
if is_pp_mode
|
||||
else (self.scheduler.running_batch,)
|
||||
)
|
||||
# Include finished requests; GPU work may still use their adapters.
|
||||
running_loras = {
|
||||
req.lora_id for batch in running_batches for req in batch.reqs
|
||||
}
|
||||
running_loras.update(r.req.lora_id for r in self.transfer_queue.queue)
|
||||
running_loras.update(req.lora_id for req in self.scheduler.waiting_queue)
|
||||
|
||||
# Then, preallocate the remaining requests if possible
|
||||
for i, decode_req in enumerate(self.queue):
|
||||
if rids_to_check is not None and decode_req.req.rid not in rids_to_check:
|
||||
@@ -1208,6 +1221,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if hisparse_req_budget <= 0:
|
||||
break
|
||||
|
||||
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
|
||||
decode_req.req, running_loras
|
||||
):
|
||||
continue
|
||||
|
||||
# Memory estimation: don't add if the projected memory cannot be met
|
||||
# TODO: add new_token ratio
|
||||
origin_input_len = self._rebootstrap_prefill_len(decode_req.req)
|
||||
@@ -1559,6 +1577,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self._num_published_destinations += 1
|
||||
preallocated_reqs.append(decode_req)
|
||||
indices_to_remove.add(i)
|
||||
if self.scheduler.enable_lora:
|
||||
running_loras.add(decode_req.req.lora_id)
|
||||
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
|
||||
|
||||
if failed_reqs:
|
||||
|
||||
@@ -3825,7 +3825,7 @@ class Scheduler(
|
||||
mamba_allocator.alloc_group_begin(len(self.waiting_queue))
|
||||
# Get requests from the waiting queue to a new prefill batch
|
||||
for req in self.waiting_queue:
|
||||
if self.enable_lora and not self._can_schedule_lora_req(req, running_loras):
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
|
||||
running_bs = len(running_batch.reqs)
|
||||
@@ -4035,7 +4035,7 @@ class Scheduler(
|
||||
|
||||
return new_batch, running_batch
|
||||
|
||||
def _can_schedule_lora_req(
|
||||
def can_schedule_lora_req(
|
||||
self, req: Req, running_loras: set[Optional[str]]
|
||||
) -> bool:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user