[PD][LoRA] Gate decode admission on adapter slots (#39332)

This commit is contained in:
Yanbin Jiang
2026-09-15 13:11:04 +08:00
committed by GitHub
parent 3f871a246c
commit ebd37705e4
5 changed files with 184 additions and 2 deletions
@@ -1188,6 +1188,19 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
- len(self.transfer_queue.queue),
)
if self.scheduler.enable_lora:
running_batches = (
self.scheduler.running_mbs
if is_pp_mode
else (self.scheduler.running_batch,)
)
# Include finished requests; GPU work may still use their adapters.
running_loras = {
req.lora_id for batch in running_batches for req in batch.reqs
}
running_loras.update(r.req.lora_id for r in self.transfer_queue.queue)
running_loras.update(req.lora_id for req in self.scheduler.waiting_queue)
# Then, preallocate the remaining requests if possible
for i, decode_req in enumerate(self.queue):
if rids_to_check is not None and decode_req.req.rid not in rids_to_check:
@@ -1208,6 +1221,11 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if hisparse_req_budget <= 0:
break
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
decode_req.req, running_loras
):
continue
# Memory estimation: don't add if the projected memory cannot be met
# TODO: add new_token ratio
origin_input_len = self._rebootstrap_prefill_len(decode_req.req)
@@ -1559,6 +1577,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self._num_published_destinations += 1
preallocated_reqs.append(decode_req)
indices_to_remove.add(i)
if self.scheduler.enable_lora:
running_loras.add(decode_req.req.lora_id)
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
if failed_reqs:
+2 -2
View File
@@ -3825,7 +3825,7 @@ class Scheduler(
mamba_allocator.alloc_group_begin(len(self.waiting_queue))
# Get requests from the waiting queue to a new prefill batch
for req in self.waiting_queue:
if self.enable_lora and not self._can_schedule_lora_req(req, running_loras):
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
running_bs = len(running_batch.reqs)
@@ -4035,7 +4035,7 @@ class Scheduler(
return new_batch, running_batch
def _can_schedule_lora_req(
def can_schedule_lora_req(
self, req: Req, running_loras: set[Optional[str]]
) -> bool:
"""