Fix LoRA usage-counter accounting across request lifecycle paths (#31808)
Co-authored-by: Yanbin Jiang <jybsuper@gmail.com>
This commit is contained in:
co-authored by
Yanbin Jiang
parent
baae0195d9
commit
b4df214948
@@ -248,6 +248,9 @@ class ReqState:
|
||||
dispatched: bool = False
|
||||
abort_sent: bool = False
|
||||
|
||||
# Prevent duplicate releases after completion or ownership transfer.
|
||||
lora_released: bool = False
|
||||
|
||||
# For streaming output
|
||||
last_output_offset: int = 0
|
||||
|
||||
@@ -661,6 +664,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
self.encoder_dispatch_ready: Dict[str, threading.Event] = {}
|
||||
self.event_loop = None
|
||||
self.asyncio_tasks = set()
|
||||
# Keep pending LoRA release tasks alive.
|
||||
self._lora_release_tasks = set()
|
||||
|
||||
# Health check
|
||||
self.server_status = ServerStatus.Starting
|
||||
@@ -883,7 +888,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
|
||||
self._init_req_state(obj, request)
|
||||
request_rids = {obj.rid} if obj.is_single else set(obj.rid)
|
||||
request_states = {
|
||||
rid: self.rid_to_state[rid]
|
||||
for rid in ([obj.rid] if obj.is_single else obj.rid)
|
||||
}
|
||||
try:
|
||||
if get_disagg().language_only:
|
||||
self._handle_epd_disaggregation_encode_request(obj)
|
||||
@@ -908,7 +916,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
yield response
|
||||
else:
|
||||
async for response in self._handle_batch_request(
|
||||
obj, request, request_rids
|
||||
obj, request, request_states
|
||||
):
|
||||
yield response
|
||||
except BaseException:
|
||||
@@ -919,7 +927,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
# request -- would otherwise leak those entries forever. Drop
|
||||
# undelivered states, but abort dispatched requests for scheduler-side
|
||||
# cleanup.
|
||||
self._release_req_states_on_failure(request_rids)
|
||||
self._release_req_states_on_failure(request_states)
|
||||
raise
|
||||
|
||||
def _detect_input_format(
|
||||
@@ -1797,36 +1805,40 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raises ValueError/HTTPException for non-stream aborts."""
|
||||
finish_reason = out["meta_info"]["finish_reason"]
|
||||
|
||||
if (
|
||||
finish_reason.get("type") == "abort"
|
||||
and finish_reason.get("status_code") == HTTPStatus.BAD_REQUEST
|
||||
):
|
||||
if not is_stream:
|
||||
raise ValueError(finish_reason["message"])
|
||||
return out
|
||||
|
||||
if finish_reason.get("type") == "abort" and finish_reason.get(
|
||||
"status_code"
|
||||
) in (
|
||||
status_code = finish_reason.get("status_code")
|
||||
if finish_reason.get("type") != "abort" or status_code not in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
):
|
||||
# Delete the key to prevent resending abort request to the scheduler and
|
||||
# to ensure aborted request state is cleaned up.
|
||||
if state.obj.rid in self.rid_to_state:
|
||||
del self.rid_to_state[state.obj.rid]
|
||||
return None
|
||||
|
||||
# Mark ongoing LoRA request as finished.
|
||||
if self.enable_lora and state.obj.lora_path:
|
||||
await self.lora_registry.release(state.obj.lora_id)
|
||||
if not is_stream:
|
||||
raise fastapi.HTTPException(
|
||||
status_code=finish_reason["status_code"],
|
||||
detail=finish_reason["message"],
|
||||
)
|
||||
return out
|
||||
# A newer request may already be using the same rid.
|
||||
if self.rid_to_state.get(state.obj.rid) is state:
|
||||
del self.rid_to_state[state.obj.rid]
|
||||
|
||||
return None
|
||||
release_task = self._release_lora_once(state)
|
||||
if release_task is not None:
|
||||
await release_task
|
||||
if not is_stream:
|
||||
if status_code == HTTPStatus.BAD_REQUEST:
|
||||
raise ValueError(finish_reason["message"])
|
||||
raise fastapi.HTTPException(
|
||||
status_code=status_code, detail=finish_reason["message"]
|
||||
)
|
||||
return out
|
||||
|
||||
def _release_lora_once(self, state: ReqState) -> Optional[asyncio.Task]:
|
||||
"""Schedule at most one LoRA release per state, returning the new task if any."""
|
||||
if state.lora_released:
|
||||
return None
|
||||
if not (self.enable_lora and state.obj.lora_path):
|
||||
return None
|
||||
state.lora_released = True
|
||||
task = asyncio.create_task(self.lora_registry.release(state.obj.lora_id))
|
||||
self._lora_release_tasks.add(task)
|
||||
task.add_done_callback(self._lora_release_tasks.discard)
|
||||
return task
|
||||
|
||||
def _wait_one_response(
|
||||
self,
|
||||
@@ -1959,11 +1971,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
async def _handle_batch_request(
|
||||
self,
|
||||
obj: Union[GenerateReqInput, EmbeddingReqInput],
|
||||
request: Optional[fastapi.Request] = None,
|
||||
request_rids: Optional[set[str]] = None,
|
||||
request: Optional[fastapi.Request],
|
||||
request_states: Dict[str, ReqState],
|
||||
):
|
||||
if request_rids is None:
|
||||
request_rids = set(obj.rid)
|
||||
batch_size = obj.batch_size
|
||||
|
||||
generators = []
|
||||
@@ -2017,6 +2027,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
# Cache the common prefix for parallel sampling
|
||||
for i in range(batch_size):
|
||||
tmp_obj = copy.copy(objs[i])
|
||||
# Warmup borrows the samples' LoRA references; only samples release
|
||||
# them. Keep LoRA on tokenized_obj for the correct prefix-cache key.
|
||||
tmp_obj.lora_path = None
|
||||
tmp_obj.lora_id = None
|
||||
tokenized_obj = copy.copy(tokenized_objs[i])
|
||||
# Ensure independent mm_items so wrap_shm_features won't mutate the original
|
||||
if hasattr(tokenized_obj, "mm_inputs") and tokenized_obj.mm_inputs:
|
||||
@@ -2029,10 +2043,20 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
tokenized_obj.sampling_params.max_new_tokens = 0
|
||||
tokenized_obj.stream = False
|
||||
self._init_req_state(tmp_obj)
|
||||
request_rids.add(tmp_obj.rid)
|
||||
request_states[tmp_obj.rid] = self.rid_to_state[tmp_obj.rid]
|
||||
await self._send_one_request(tokenized_obj)
|
||||
await self._wait_one_response(tmp_obj, request).__anext__()
|
||||
|
||||
# Retain initial states until all warmups succeed so errors can
|
||||
# release their LoRA references. Sampling requests take ownership below.
|
||||
initial_req_states = {}
|
||||
for initial_rid in obj.rid if isinstance(obj.rid, list) else [obj.rid]:
|
||||
initial_state = request_states[initial_rid]
|
||||
if self.rid_to_state.get(initial_rid) is initial_state:
|
||||
del self.rid_to_state[initial_rid]
|
||||
initial_state.lora_released = True
|
||||
initial_req_states[initial_rid] = initial_state
|
||||
|
||||
# Expand requests, assign new rids for them, and send them
|
||||
for i in range(batch_size):
|
||||
for _ in range(obj.parallel_sample_num):
|
||||
@@ -2046,8 +2070,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
]
|
||||
tokenized_obj.rid = tmp_obj.regenerate_rid()
|
||||
self._init_req_state(tmp_obj)
|
||||
request_rids.add(tmp_obj.rid)
|
||||
state = self.rid_to_state[tmp_obj.rid]
|
||||
request_states[tmp_obj.rid] = state
|
||||
tokenized_obj.time_stats = state.time_stats
|
||||
if tmp_obj.return_prompt_token_ids:
|
||||
state.prompt_token_ids = list(tokenized_objs[i].input_ids)
|
||||
@@ -2055,8 +2079,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
generators.append(self._wait_one_response(tmp_obj, request))
|
||||
rids.append(tmp_obj.rid)
|
||||
|
||||
self.rid_to_state[objs[i].rid].time_stats.set_finished_time()
|
||||
del self.rid_to_state[objs[i].rid]
|
||||
if objs[i].rid in initial_req_states:
|
||||
initial_req_states[objs[i].rid].time_stats.set_finished_time()
|
||||
|
||||
# Wait for all requests
|
||||
is_stream = hasattr(obj, "stream") and obj.stream
|
||||
@@ -2637,9 +2661,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
|
||||
del self.rid_to_state[rid]
|
||||
|
||||
# Mark ongoing LoRA request as finished.
|
||||
if self.enable_lora and state.obj.lora_path:
|
||||
asyncio.create_task(self.lora_registry.release(state.obj.lora_id))
|
||||
self._release_lora_once(state)
|
||||
|
||||
if out_dict is not None:
|
||||
state.out_list.append(out_dict)
|
||||
@@ -3466,6 +3488,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
out["prompt_token_ids"] = state.prompt_token_ids
|
||||
del self.rid_to_state[recv_obj.rid]
|
||||
|
||||
self._release_lora_once(state)
|
||||
|
||||
state.out_list.append(out)
|
||||
state.event.set()
|
||||
|
||||
@@ -3665,15 +3689,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
time_stats.init_trace_ctx(rid, bootstrap_room, external_trace_header)
|
||||
time_stats.set_created_time(created_time)
|
||||
|
||||
def _release_req_states_on_failure(self, rids: Iterable[str]):
|
||||
def _release_req_states_on_failure(self, request_states: Dict[str, ReqState]):
|
||||
"""Release rid_to_state entries created for a failed handler.
|
||||
|
||||
Undelivered states are removed locally. Dispatched requests are aborted
|
||||
and retained until the scheduler response removes them.
|
||||
"""
|
||||
for rid in rids:
|
||||
state = self.rid_to_state.get(rid)
|
||||
if state is not None:
|
||||
for rid, state in request_states.items():
|
||||
current_state = self.rid_to_state.get(rid)
|
||||
if current_state is not None:
|
||||
# A completed request's rid may already belong to a newer request.
|
||||
if current_state is not state:
|
||||
continue
|
||||
if state.dispatched:
|
||||
try:
|
||||
self.abort_request(rid)
|
||||
@@ -3683,6 +3710,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
else:
|
||||
del self.rid_to_state[rid]
|
||||
self._release_lora_once(state)
|
||||
dispatch_ready = self.encoder_dispatch_ready.pop(rid, None)
|
||||
if dispatch_ready is not None:
|
||||
dispatch_ready.set()
|
||||
|
||||
@@ -17,9 +17,11 @@ Covers:
|
||||
import asyncio
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import msgspec
|
||||
from fastapi import HTTPException
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
@@ -27,6 +29,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode # noqa: E402
|
||||
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry # noqa: E402
|
||||
from sglang.srt.managers.io_struct import ( # noqa: E402
|
||||
AbortReq,
|
||||
BatchStrOutput,
|
||||
@@ -313,16 +316,23 @@ class TestRidToStateCleanupOnAbort(CustomTestCase):
|
||||
"""Test that _handle_abort_req removes rid from rid_to_state."""
|
||||
|
||||
def test_abort_removes_rid_from_state(self):
|
||||
"""After _handle_abort_req, rid should be removed from rid_to_state."""
|
||||
tm = _make_tokenizer_manager(self)
|
||||
rid = "abort_test_rid"
|
||||
state = _make_req_state(rid)
|
||||
tm.rid_to_state[rid] = state
|
||||
"""An abort must release its adapter even when no response waiter remains."""
|
||||
|
||||
abort_req = _make_abort_req(rid)
|
||||
tm._handle_abort_req(abort_req)
|
||||
async def drive():
|
||||
tm = _make_tm_for_generate(self, enable_lora=True)
|
||||
obj = GenerateReqInput(
|
||||
text="prompt", rid="abort_test_rid", lora_path="adapter"
|
||||
)
|
||||
obj.normalize_batch_and_arguments()
|
||||
tm._init_req_state(obj)
|
||||
await tm._validate_and_resolve_lora(obj)
|
||||
|
||||
self.assertNotIn(rid, tm.rid_to_state)
|
||||
tm._handle_abort_req(_make_abort_req(obj.rid))
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertNotIn(obj.rid, tm.rid_to_state)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0)
|
||||
|
||||
asyncio.run(drive())
|
||||
|
||||
def test_abort_allows_resubmit_same_rid(self):
|
||||
"""After abort, _init_req_state should accept the same rid again."""
|
||||
@@ -663,7 +673,7 @@ class _DummyAsyncCM:
|
||||
return False
|
||||
|
||||
|
||||
def _make_tm_for_generate(case) -> TokenizerManager:
|
||||
def _make_tm_for_generate(case, enable_lora=False) -> TokenizerManager:
|
||||
"""Augment the mocked TokenizerManager with what generate_request needs."""
|
||||
tm = _make_tokenizer_manager(case)
|
||||
tm.server_args.language_only = False
|
||||
@@ -677,7 +687,17 @@ def _make_tm_for_generate(case) -> TokenizerManager:
|
||||
tm.is_pause_cond = asyncio.Condition()
|
||||
tm.model_update_lock = Mock()
|
||||
tm.model_update_lock.reader_lock = _DummyAsyncCM()
|
||||
tm._validate_and_resolve_lora = AsyncMock(return_value=None)
|
||||
tm.request_metrics_exporter_manager = Mock()
|
||||
tm.request_metrics_exporter_manager.exporter_enabled.return_value = False
|
||||
tm._lora_release_tasks = set()
|
||||
if enable_lora:
|
||||
tm.enable_lora = True
|
||||
tm.server_args.max_loaded_loras = None
|
||||
tm.lora_registry = LoRARegistry(
|
||||
[LoRARef(lora_id="adapter", lora_name="adapter")]
|
||||
)
|
||||
else:
|
||||
tm._validate_and_resolve_lora = AsyncMock(return_value=None)
|
||||
return tm
|
||||
|
||||
|
||||
@@ -703,7 +723,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
tm = _make_tokenizer_manager(self)
|
||||
rid = "d_single"
|
||||
tm.rid_to_state[rid] = _make_req_state(rid)
|
||||
tm._release_req_states_on_failure([rid])
|
||||
tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]})
|
||||
self.assertNotIn(rid, tm.rid_to_state)
|
||||
|
||||
def test_undelivered_batch_removes_all(self):
|
||||
@@ -711,7 +731,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
rids = ["d0", "d1", "d2"]
|
||||
for r in rids:
|
||||
tm.rid_to_state[r] = _make_req_state(r)
|
||||
tm._release_req_states_on_failure(rids)
|
||||
tm._release_req_states_on_failure(dict(tm.rid_to_state))
|
||||
for r in rids:
|
||||
self.assertNotIn(r, tm.rid_to_state)
|
||||
|
||||
@@ -719,7 +739,12 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
"""A rid that is no longer present must not raise."""
|
||||
tm = _make_tokenizer_manager(self)
|
||||
tm.rid_to_state["p1"] = _make_req_state("p1")
|
||||
tm._release_req_states_on_failure(["p1", "already_gone"])
|
||||
tm._release_req_states_on_failure(
|
||||
{
|
||||
"p1": tm.rid_to_state["p1"],
|
||||
"already_gone": _make_req_state("already_gone"),
|
||||
}
|
||||
)
|
||||
self.assertNotIn("p1", tm.rid_to_state)
|
||||
|
||||
def test_dispatched_single_is_aborted_and_state_kept(self):
|
||||
@@ -732,8 +757,8 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
state = _make_req_state(rid)
|
||||
state.dispatched = True
|
||||
tm.rid_to_state[rid] = state
|
||||
tm._release_req_states_on_failure([rid])
|
||||
tm._release_req_states_on_failure([rid])
|
||||
tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]})
|
||||
tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]})
|
||||
|
||||
sent = [c.args[0] for c in tm._dispatch_to_scheduler.call_args_list]
|
||||
self.assertEqual(
|
||||
@@ -753,7 +778,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
live.dispatched = True
|
||||
tm.rid_to_state[delivered] = live
|
||||
tm.rid_to_state[undelivered] = _make_req_state(undelivered)
|
||||
tm._release_req_states_on_failure([delivered, undelivered])
|
||||
tm._release_req_states_on_failure(dict(tm.rid_to_state))
|
||||
|
||||
sent = [c.args[0] for c in tm._dispatch_to_scheduler.call_args_list]
|
||||
self.assertEqual([type(m) for m in sent], [AbortReq])
|
||||
@@ -772,12 +797,56 @@ class TestReleaseReqStatesOnFailure(CustomTestCase):
|
||||
tm.rid_to_state[undelivered] = _make_req_state(undelivered)
|
||||
|
||||
with self.assertLogs(level="ERROR"):
|
||||
tm._release_req_states_on_failure([delivered, undelivered])
|
||||
tm._release_req_states_on_failure(dict(tm.rid_to_state))
|
||||
|
||||
self.assertIn(delivered, tm.rid_to_state)
|
||||
self.assertFalse(live.abort_sent)
|
||||
self.assertNotIn(undelivered, tm.rid_to_state)
|
||||
|
||||
def test_completed_sample_reuse_survives_sibling_cleanup(self):
|
||||
"""Batch cleanup must preserve a reused sample RID, adapter ref and encoder gate."""
|
||||
|
||||
async def drive(dispatched):
|
||||
tm = _make_tm_for_generate(self, enable_lora=True)
|
||||
tm._dispatch_to_scheduler = Mock()
|
||||
samples = [
|
||||
GenerateReqInput(text="sample", lora_path="adapter") for _ in range(2)
|
||||
]
|
||||
for sample in samples:
|
||||
sample.normalize_batch_and_arguments()
|
||||
tm._init_req_state(sample)
|
||||
await tm._validate_and_resolve_lora(sample)
|
||||
request_states = dict(tm.rid_to_state)
|
||||
rid = samples[0].rid
|
||||
await tm._handle_batch_output(_make_batch_str_output(rid))
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
|
||||
replacement = GenerateReqInput(text="new", rid=rid, lora_path="adapter")
|
||||
replacement.normalize_batch_and_arguments()
|
||||
tm._init_req_state(replacement)
|
||||
await tm._validate_and_resolve_lora(replacement)
|
||||
replacement_state = tm.rid_to_state[rid]
|
||||
replacement_state.dispatched = dispatched
|
||||
gate = asyncio.Event()
|
||||
tm.encoder_dispatch_ready[rid] = gate
|
||||
|
||||
tm._release_req_states_on_failure(request_states)
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertIs(tm.rid_to_state.get(rid), replacement_state)
|
||||
self.assertNotIn(samples[1].rid, tm.rid_to_state)
|
||||
self.assertIs(tm.encoder_dispatch_ready.get(rid), gate)
|
||||
self.assertFalse(gate.is_set())
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 1)
|
||||
tm._dispatch_to_scheduler.assert_not_called()
|
||||
|
||||
await tm._handle_batch_output(_make_batch_str_output(rid))
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0)
|
||||
|
||||
for dispatched in (False, True):
|
||||
with self.subTest(dispatched=dispatched):
|
||||
asyncio.run(drive(dispatched))
|
||||
|
||||
|
||||
class TestParallelStreamTaskCleanup(CustomTestCase):
|
||||
def test_failing_choice_cancels_and_closes_sibling_waiters(self):
|
||||
@@ -843,24 +912,22 @@ class TestGenerateRequestCleanupOnDispatchFailure(CustomTestCase):
|
||||
"""
|
||||
|
||||
def test_single_failure_before_dispatch_cleans_up(self):
|
||||
tm = _make_tm_for_generate(self)
|
||||
rid = "single_overlen"
|
||||
obj = _make_generate_obj(rid, is_single=True)
|
||||
# Simulate over-length rejection during tokenization/validation.
|
||||
"""Validation failure must return the adapter reference acquired earlier."""
|
||||
tm = _make_tm_for_generate(self, enable_lora=True)
|
||||
obj = GenerateReqInput(text="prompt", rid="single_overlen", lora_path="adapter")
|
||||
tm._tokenize_one_request = AsyncMock(side_effect=ValueError("input too long"))
|
||||
tm._send_one_request = Mock()
|
||||
|
||||
async def drive():
|
||||
await tm.generate_request(obj).__anext__()
|
||||
with self.assertRaisesRegex(ValueError, "input too long"):
|
||||
await tm.generate_request(obj).__anext__()
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
asyncio.run(drive())
|
||||
|
||||
# Got past _init_req_state (which created the entry) ...
|
||||
asyncio.run(drive())
|
||||
tm._tokenize_one_request.assert_awaited_once()
|
||||
tm._send_one_request.assert_not_called()
|
||||
# ... and the entry was cleaned up rather than leaked.
|
||||
self.assertNotIn(rid, tm.rid_to_state)
|
||||
self.assertNotIn(obj.rid, tm.rid_to_state)
|
||||
|
||||
def test_batch_failure_before_dispatch_cleans_up_all(self):
|
||||
tm = _make_tm_for_generate(self)
|
||||
@@ -998,5 +1065,131 @@ class TestDisconnectAfterDispatchAbortsRequest(CustomTestCase):
|
||||
self.assertIn(rid, tm.rid_to_state)
|
||||
|
||||
|
||||
class TestLoRARequestLifecycle(CustomTestCase):
|
||||
def test_abort_waiter_preserves_reused_rid(self):
|
||||
"""An old abort must neither erase a new request nor release its reference."""
|
||||
|
||||
async def drive(status_code, expected_exception):
|
||||
tm = _make_tm_for_generate(self, enable_lora=True)
|
||||
obj = GenerateReqInput(text="old", rid="reused", lora_path="adapter")
|
||||
tm._tokenize_one_request = AsyncMock(
|
||||
side_effect=lambda req: SimpleNamespace(rid=req.rid)
|
||||
)
|
||||
dispatched = asyncio.Event()
|
||||
|
||||
async def send(tokenized):
|
||||
tm._mark_state_dispatched(tokenized.rid)
|
||||
dispatched.set()
|
||||
|
||||
tm._send_one_request = send
|
||||
generator = tm.generate_request(obj)
|
||||
waiter = asyncio.create_task(generator.__anext__())
|
||||
await dispatched.wait()
|
||||
|
||||
abort = _make_abort_req(obj.rid)
|
||||
abort.finished_reason["status_code"] = status_code
|
||||
tm._handle_abort_req(abort)
|
||||
# Reuse the ID before the old waiter resumes on its abort notification.
|
||||
replacement = GenerateReqInput(text="new", rid=obj.rid, lora_path="adapter")
|
||||
replacement.normalize_batch_and_arguments()
|
||||
tm._init_req_state(replacement)
|
||||
await tm._validate_and_resolve_lora(replacement)
|
||||
replacement_state = tm.rid_to_state[obj.rid]
|
||||
|
||||
with self.assertRaises(expected_exception) as ctx:
|
||||
await waiter
|
||||
if expected_exception is HTTPException:
|
||||
self.assertEqual(ctx.exception.status_code, status_code)
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertIs(tm.rid_to_state.get(obj.rid), replacement_state)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 1)
|
||||
|
||||
await tm._handle_batch_output(_make_batch_str_output(obj.rid))
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0)
|
||||
await generator.aclose()
|
||||
|
||||
for status_code, expected_exception in (
|
||||
(400, ValueError),
|
||||
(503, HTTPException),
|
||||
):
|
||||
with self.subTest(status_code=status_code):
|
||||
asyncio.run(drive(status_code, expected_exception))
|
||||
|
||||
async def _drive_parallel_request(self, fail_second_warmup=False):
|
||||
tm = _make_tm_for_generate(self, enable_lora=True)
|
||||
obj = GenerateReqInput(
|
||||
text=["first", "second"],
|
||||
lora_path="adapter",
|
||||
sampling_params={"n": 2, "max_new_tokens": 4},
|
||||
)
|
||||
tm._tokenize_one_request = AsyncMock(
|
||||
side_effect=lambda req: SimpleNamespace(
|
||||
rid=req.rid,
|
||||
lora_id=req.lora_id,
|
||||
input_ids=[1],
|
||||
mm_inputs=None,
|
||||
time_stats=tm.rid_to_state[req.rid].time_stats,
|
||||
sampling_params=SimpleNamespace(max_new_tokens=4),
|
||||
)
|
||||
)
|
||||
sent = asyncio.Queue()
|
||||
|
||||
async def send(tokenized):
|
||||
tm._mark_state_dispatched(tokenized.rid)
|
||||
sent.put_nowait(tokenized)
|
||||
|
||||
tm._send_one_request = send
|
||||
generator = tm.generate_request(obj)
|
||||
response = asyncio.create_task(generator.__anext__())
|
||||
try:
|
||||
for i in range(2):
|
||||
warmup = await asyncio.wait_for(sent.get(), timeout=1)
|
||||
self.assertEqual(warmup.lora_id, "adapter")
|
||||
self.assertEqual(warmup.sampling_params.max_new_tokens, 0)
|
||||
reason = None
|
||||
if fail_second_warmup and i == 1:
|
||||
reason = {
|
||||
"type": "abort",
|
||||
"status_code": 400,
|
||||
"message": "invalid grammar",
|
||||
}
|
||||
await tm._handle_batch_output(
|
||||
_make_batch_str_output(warmup.rid, reason)
|
||||
)
|
||||
|
||||
if fail_second_warmup:
|
||||
with self.assertRaisesRegex(ValueError, "invalid grammar"):
|
||||
await response
|
||||
self.assertTrue(sent.empty()) # No samples were dispatched.
|
||||
else:
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 4)
|
||||
for remaining in (3, 2, 1, 0):
|
||||
sample = await asyncio.wait_for(sent.get(), timeout=1)
|
||||
await tm._handle_batch_output(_make_batch_str_output(sample.rid))
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(
|
||||
tm.lora_registry._counters["adapter"].value(), remaining
|
||||
)
|
||||
self.assertEqual(len(await response), 4)
|
||||
|
||||
await asyncio.gather(*tm._lora_release_tasks)
|
||||
self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0)
|
||||
self.assertFalse(tm.rid_to_state)
|
||||
finally:
|
||||
response.cancel()
|
||||
await asyncio.gather(response, return_exceptions=True)
|
||||
await generator.aclose()
|
||||
|
||||
def test_parallel_warmups_do_not_release_sample_references(self):
|
||||
"""Prefix warmups must not let an adapter unload while samples use it."""
|
||||
asyncio.run(self._drive_parallel_request())
|
||||
|
||||
def test_second_parallel_warmup_failure_releases_all_references(self):
|
||||
"""A failed warmup must return references even after earlier warmups finish."""
|
||||
asyncio.run(self._drive_parallel_request(fail_second_warmup=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
Reference in New Issue
Block a user