Fix LoRA usage-counter accounting across request lifecycle paths (#31808)

Co-authored-by: Yanbin Jiang <jybsuper@gmail.com>
This commit is contained in:
Shafeeq Ibraheem
2026-09-30 18:37:53 -07:00
committed by GitHub
co-authored by Yanbin Jiang
parent baae0195d9
commit b4df214948
2 changed files with 292 additions and 71 deletions
+71 -43
View File
@@ -248,6 +248,9 @@ class ReqState:
dispatched: bool = False
abort_sent: bool = False
# Prevent duplicate releases after completion or ownership transfer.
lora_released: bool = False
# For streaming output
last_output_offset: int = 0
@@ -661,6 +664,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
self.encoder_dispatch_ready: Dict[str, threading.Event] = {}
self.event_loop = None
self.asyncio_tasks = set()
# Keep pending LoRA release tasks alive.
self._lora_release_tasks = set()
# Health check
self.server_status = ServerStatus.Starting
@@ -883,7 +888,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
self._init_req_state(obj, request)
request_rids = {obj.rid} if obj.is_single else set(obj.rid)
request_states = {
rid: self.rid_to_state[rid]
for rid in ([obj.rid] if obj.is_single else obj.rid)
}
try:
if get_disagg().language_only:
self._handle_epd_disaggregation_encode_request(obj)
@@ -908,7 +916,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
yield response
else:
async for response in self._handle_batch_request(
obj, request, request_rids
obj, request, request_states
):
yield response
except BaseException:
@@ -919,7 +927,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
# request -- would otherwise leak those entries forever. Drop
# undelivered states, but abort dispatched requests for scheduler-side
# cleanup.
self._release_req_states_on_failure(request_rids)
self._release_req_states_on_failure(request_states)
raise
def _detect_input_format(
@@ -1797,36 +1805,40 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raises ValueError/HTTPException for non-stream aborts."""
finish_reason = out["meta_info"]["finish_reason"]
if (
finish_reason.get("type") == "abort"
and finish_reason.get("status_code") == HTTPStatus.BAD_REQUEST
):
if not is_stream:
raise ValueError(finish_reason["message"])
return out
if finish_reason.get("type") == "abort" and finish_reason.get(
"status_code"
) in (
status_code = finish_reason.get("status_code")
if finish_reason.get("type") != "abort" or status_code not in (
HTTPStatus.BAD_REQUEST,
HTTPStatus.SERVICE_UNAVAILABLE,
HTTPStatus.INTERNAL_SERVER_ERROR,
):
# Delete the key to prevent resending abort request to the scheduler and
# to ensure aborted request state is cleaned up.
if state.obj.rid in self.rid_to_state:
del self.rid_to_state[state.obj.rid]
return None
# Mark ongoing LoRA request as finished.
if self.enable_lora and state.obj.lora_path:
await self.lora_registry.release(state.obj.lora_id)
if not is_stream:
raise fastapi.HTTPException(
status_code=finish_reason["status_code"],
detail=finish_reason["message"],
)
return out
# A newer request may already be using the same rid.
if self.rid_to_state.get(state.obj.rid) is state:
del self.rid_to_state[state.obj.rid]
return None
release_task = self._release_lora_once(state)
if release_task is not None:
await release_task
if not is_stream:
if status_code == HTTPStatus.BAD_REQUEST:
raise ValueError(finish_reason["message"])
raise fastapi.HTTPException(
status_code=status_code, detail=finish_reason["message"]
)
return out
def _release_lora_once(self, state: ReqState) -> Optional[asyncio.Task]:
"""Schedule at most one LoRA release per state, returning the new task if any."""
if state.lora_released:
return None
if not (self.enable_lora and state.obj.lora_path):
return None
state.lora_released = True
task = asyncio.create_task(self.lora_registry.release(state.obj.lora_id))
self._lora_release_tasks.add(task)
task.add_done_callback(self._lora_release_tasks.discard)
return task
def _wait_one_response(
self,
@@ -1959,11 +1971,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
async def _handle_batch_request(
self,
obj: Union[GenerateReqInput, EmbeddingReqInput],
request: Optional[fastapi.Request] = None,
request_rids: Optional[set[str]] = None,
request: Optional[fastapi.Request],
request_states: Dict[str, ReqState],
):
if request_rids is None:
request_rids = set(obj.rid)
batch_size = obj.batch_size
generators = []
@@ -2017,6 +2027,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
# Cache the common prefix for parallel sampling
for i in range(batch_size):
tmp_obj = copy.copy(objs[i])
# Warmup borrows the samples' LoRA references; only samples release
# them. Keep LoRA on tokenized_obj for the correct prefix-cache key.
tmp_obj.lora_path = None
tmp_obj.lora_id = None
tokenized_obj = copy.copy(tokenized_objs[i])
# Ensure independent mm_items so wrap_shm_features won't mutate the original
if hasattr(tokenized_obj, "mm_inputs") and tokenized_obj.mm_inputs:
@@ -2029,10 +2043,20 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
tokenized_obj.sampling_params.max_new_tokens = 0
tokenized_obj.stream = False
self._init_req_state(tmp_obj)
request_rids.add(tmp_obj.rid)
request_states[tmp_obj.rid] = self.rid_to_state[tmp_obj.rid]
await self._send_one_request(tokenized_obj)
await self._wait_one_response(tmp_obj, request).__anext__()
# Retain initial states until all warmups succeed so errors can
# release their LoRA references. Sampling requests take ownership below.
initial_req_states = {}
for initial_rid in obj.rid if isinstance(obj.rid, list) else [obj.rid]:
initial_state = request_states[initial_rid]
if self.rid_to_state.get(initial_rid) is initial_state:
del self.rid_to_state[initial_rid]
initial_state.lora_released = True
initial_req_states[initial_rid] = initial_state
# Expand requests, assign new rids for them, and send them
for i in range(batch_size):
for _ in range(obj.parallel_sample_num):
@@ -2046,8 +2070,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
]
tokenized_obj.rid = tmp_obj.regenerate_rid()
self._init_req_state(tmp_obj)
request_rids.add(tmp_obj.rid)
state = self.rid_to_state[tmp_obj.rid]
request_states[tmp_obj.rid] = state
tokenized_obj.time_stats = state.time_stats
if tmp_obj.return_prompt_token_ids:
state.prompt_token_ids = list(tokenized_objs[i].input_ids)
@@ -2055,8 +2079,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
generators.append(self._wait_one_response(tmp_obj, request))
rids.append(tmp_obj.rid)
self.rid_to_state[objs[i].rid].time_stats.set_finished_time()
del self.rid_to_state[objs[i].rid]
if objs[i].rid in initial_req_states:
initial_req_states[objs[i].rid].time_stats.set_finished_time()
# Wait for all requests
is_stream = hasattr(obj, "stream") and obj.stream
@@ -2637,9 +2661,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
del self.rid_to_state[rid]
# Mark ongoing LoRA request as finished.
if self.enable_lora and state.obj.lora_path:
asyncio.create_task(self.lora_registry.release(state.obj.lora_id))
self._release_lora_once(state)
if out_dict is not None:
state.out_list.append(out_dict)
@@ -3466,6 +3488,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
out["prompt_token_ids"] = state.prompt_token_ids
del self.rid_to_state[recv_obj.rid]
self._release_lora_once(state)
state.out_list.append(out)
state.event.set()
@@ -3665,15 +3689,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
time_stats.init_trace_ctx(rid, bootstrap_room, external_trace_header)
time_stats.set_created_time(created_time)
def _release_req_states_on_failure(self, rids: Iterable[str]):
def _release_req_states_on_failure(self, request_states: Dict[str, ReqState]):
"""Release rid_to_state entries created for a failed handler.
Undelivered states are removed locally. Dispatched requests are aborted
and retained until the scheduler response removes them.
"""
for rid in rids:
state = self.rid_to_state.get(rid)
if state is not None:
for rid, state in request_states.items():
current_state = self.rid_to_state.get(rid)
if current_state is not None:
# A completed request's rid may already belong to a newer request.
if current_state is not state:
continue
if state.dispatched:
try:
self.abort_request(rid)
@@ -3683,6 +3710,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
else:
del self.rid_to_state[rid]
self._release_lora_once(state)
dispatch_ready = self.encoder_dispatch_ready.pop(rid, None)
if dispatch_ready is not None:
dispatch_ready.set()