diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 797e8a6180..2ccae590ef 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -248,6 +248,9 @@ class ReqState: dispatched: bool = False abort_sent: bool = False + # Prevent duplicate releases after completion or ownership transfer. + lora_released: bool = False + # For streaming output last_output_offset: int = 0 @@ -661,6 +664,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): self.encoder_dispatch_ready: Dict[str, threading.Event] = {} self.event_loop = None self.asyncio_tasks = set() + # Keep pending LoRA release tasks alive. + self._lora_release_tasks = set() # Health check self.server_status = ServerStatus.Starting @@ -883,7 +888,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) self._init_req_state(obj, request) - request_rids = {obj.rid} if obj.is_single else set(obj.rid) + request_states = { + rid: self.rid_to_state[rid] + for rid in ([obj.rid] if obj.is_single else obj.rid) + } try: if get_disagg().language_only: self._handle_epd_disaggregation_encode_request(obj) @@ -908,7 +916,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): yield response else: async for response in self._handle_batch_request( - obj, request, request_rids + obj, request, request_states ): yield response except BaseException: @@ -919,7 +927,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # request -- would otherwise leak those entries forever. Drop # undelivered states, but abort dispatched requests for scheduler-side # cleanup. - self._release_req_states_on_failure(request_rids) + self._release_req_states_on_failure(request_states) raise def _detect_input_format( @@ -1797,36 +1805,40 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): raises ValueError/HTTPException for non-stream aborts.""" finish_reason = out["meta_info"]["finish_reason"] - if ( - finish_reason.get("type") == "abort" - and finish_reason.get("status_code") == HTTPStatus.BAD_REQUEST - ): - if not is_stream: - raise ValueError(finish_reason["message"]) - return out - - if finish_reason.get("type") == "abort" and finish_reason.get( - "status_code" - ) in ( + status_code = finish_reason.get("status_code") + if finish_reason.get("type") != "abort" or status_code not in ( + HTTPStatus.BAD_REQUEST, HTTPStatus.SERVICE_UNAVAILABLE, HTTPStatus.INTERNAL_SERVER_ERROR, ): - # Delete the key to prevent resending abort request to the scheduler and - # to ensure aborted request state is cleaned up. - if state.obj.rid in self.rid_to_state: - del self.rid_to_state[state.obj.rid] + return None - # Mark ongoing LoRA request as finished. - if self.enable_lora and state.obj.lora_path: - await self.lora_registry.release(state.obj.lora_id) - if not is_stream: - raise fastapi.HTTPException( - status_code=finish_reason["status_code"], - detail=finish_reason["message"], - ) - return out + # A newer request may already be using the same rid. + if self.rid_to_state.get(state.obj.rid) is state: + del self.rid_to_state[state.obj.rid] - return None + release_task = self._release_lora_once(state) + if release_task is not None: + await release_task + if not is_stream: + if status_code == HTTPStatus.BAD_REQUEST: + raise ValueError(finish_reason["message"]) + raise fastapi.HTTPException( + status_code=status_code, detail=finish_reason["message"] + ) + return out + + def _release_lora_once(self, state: ReqState) -> Optional[asyncio.Task]: + """Schedule at most one LoRA release per state, returning the new task if any.""" + if state.lora_released: + return None + if not (self.enable_lora and state.obj.lora_path): + return None + state.lora_released = True + task = asyncio.create_task(self.lora_registry.release(state.obj.lora_id)) + self._lora_release_tasks.add(task) + task.add_done_callback(self._lora_release_tasks.discard) + return task def _wait_one_response( self, @@ -1959,11 +1971,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): async def _handle_batch_request( self, obj: Union[GenerateReqInput, EmbeddingReqInput], - request: Optional[fastapi.Request] = None, - request_rids: Optional[set[str]] = None, + request: Optional[fastapi.Request], + request_states: Dict[str, ReqState], ): - if request_rids is None: - request_rids = set(obj.rid) batch_size = obj.batch_size generators = [] @@ -2017,6 +2027,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Cache the common prefix for parallel sampling for i in range(batch_size): tmp_obj = copy.copy(objs[i]) + # Warmup borrows the samples' LoRA references; only samples release + # them. Keep LoRA on tokenized_obj for the correct prefix-cache key. + tmp_obj.lora_path = None + tmp_obj.lora_id = None tokenized_obj = copy.copy(tokenized_objs[i]) # Ensure independent mm_items so wrap_shm_features won't mutate the original if hasattr(tokenized_obj, "mm_inputs") and tokenized_obj.mm_inputs: @@ -2029,10 +2043,20 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): tokenized_obj.sampling_params.max_new_tokens = 0 tokenized_obj.stream = False self._init_req_state(tmp_obj) - request_rids.add(tmp_obj.rid) + request_states[tmp_obj.rid] = self.rid_to_state[tmp_obj.rid] await self._send_one_request(tokenized_obj) await self._wait_one_response(tmp_obj, request).__anext__() + # Retain initial states until all warmups succeed so errors can + # release their LoRA references. Sampling requests take ownership below. + initial_req_states = {} + for initial_rid in obj.rid if isinstance(obj.rid, list) else [obj.rid]: + initial_state = request_states[initial_rid] + if self.rid_to_state.get(initial_rid) is initial_state: + del self.rid_to_state[initial_rid] + initial_state.lora_released = True + initial_req_states[initial_rid] = initial_state + # Expand requests, assign new rids for them, and send them for i in range(batch_size): for _ in range(obj.parallel_sample_num): @@ -2046,8 +2070,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ] tokenized_obj.rid = tmp_obj.regenerate_rid() self._init_req_state(tmp_obj) - request_rids.add(tmp_obj.rid) state = self.rid_to_state[tmp_obj.rid] + request_states[tmp_obj.rid] = state tokenized_obj.time_stats = state.time_stats if tmp_obj.return_prompt_token_ids: state.prompt_token_ids = list(tokenized_objs[i].input_ids) @@ -2055,8 +2079,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): generators.append(self._wait_one_response(tmp_obj, request)) rids.append(tmp_obj.rid) - self.rid_to_state[objs[i].rid].time_stats.set_finished_time() - del self.rid_to_state[objs[i].rid] + if objs[i].rid in initial_req_states: + initial_req_states[objs[i].rid].time_stats.set_finished_time() # Wait for all requests is_stream = hasattr(obj, "stream") and obj.stream @@ -2637,9 +2661,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): del self.rid_to_state[rid] - # Mark ongoing LoRA request as finished. - if self.enable_lora and state.obj.lora_path: - asyncio.create_task(self.lora_registry.release(state.obj.lora_id)) + self._release_lora_once(state) if out_dict is not None: state.out_list.append(out_dict) @@ -3466,6 +3488,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): out["prompt_token_ids"] = state.prompt_token_ids del self.rid_to_state[recv_obj.rid] + self._release_lora_once(state) + state.out_list.append(out) state.event.set() @@ -3665,15 +3689,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): time_stats.init_trace_ctx(rid, bootstrap_room, external_trace_header) time_stats.set_created_time(created_time) - def _release_req_states_on_failure(self, rids: Iterable[str]): + def _release_req_states_on_failure(self, request_states: Dict[str, ReqState]): """Release rid_to_state entries created for a failed handler. Undelivered states are removed locally. Dispatched requests are aborted and retained until the scheduler response removes them. """ - for rid in rids: - state = self.rid_to_state.get(rid) - if state is not None: + for rid, state in request_states.items(): + current_state = self.rid_to_state.get(rid) + if current_state is not None: + # A completed request's rid may already belong to a newer request. + if current_state is not state: + continue if state.dispatched: try: self.abort_request(rid) @@ -3683,6 +3710,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) else: del self.rid_to_state[rid] + self._release_lora_once(state) dispatch_ready = self.encoder_dispatch_ready.pop(rid, None) if dispatch_ready is not None: dispatch_ready.set() diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index 0b1e2eb54e..3c9df4ccba 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -17,9 +17,11 @@ Covers: import asyncio import time import unittest +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, Mock, patch import msgspec +from fastapi import HTTPException from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -27,6 +29,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() from sglang.srt.disaggregation.utils import DisaggregationMode # noqa: E402 +from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry # noqa: E402 from sglang.srt.managers.io_struct import ( # noqa: E402 AbortReq, BatchStrOutput, @@ -313,16 +316,23 @@ class TestRidToStateCleanupOnAbort(CustomTestCase): """Test that _handle_abort_req removes rid from rid_to_state.""" def test_abort_removes_rid_from_state(self): - """After _handle_abort_req, rid should be removed from rid_to_state.""" - tm = _make_tokenizer_manager(self) - rid = "abort_test_rid" - state = _make_req_state(rid) - tm.rid_to_state[rid] = state + """An abort must release its adapter even when no response waiter remains.""" - abort_req = _make_abort_req(rid) - tm._handle_abort_req(abort_req) + async def drive(): + tm = _make_tm_for_generate(self, enable_lora=True) + obj = GenerateReqInput( + text="prompt", rid="abort_test_rid", lora_path="adapter" + ) + obj.normalize_batch_and_arguments() + tm._init_req_state(obj) + await tm._validate_and_resolve_lora(obj) - self.assertNotIn(rid, tm.rid_to_state) + tm._handle_abort_req(_make_abort_req(obj.rid)) + await asyncio.gather(*tm._lora_release_tasks) + self.assertNotIn(obj.rid, tm.rid_to_state) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0) + + asyncio.run(drive()) def test_abort_allows_resubmit_same_rid(self): """After abort, _init_req_state should accept the same rid again.""" @@ -663,7 +673,7 @@ class _DummyAsyncCM: return False -def _make_tm_for_generate(case) -> TokenizerManager: +def _make_tm_for_generate(case, enable_lora=False) -> TokenizerManager: """Augment the mocked TokenizerManager with what generate_request needs.""" tm = _make_tokenizer_manager(case) tm.server_args.language_only = False @@ -677,7 +687,17 @@ def _make_tm_for_generate(case) -> TokenizerManager: tm.is_pause_cond = asyncio.Condition() tm.model_update_lock = Mock() tm.model_update_lock.reader_lock = _DummyAsyncCM() - tm._validate_and_resolve_lora = AsyncMock(return_value=None) + tm.request_metrics_exporter_manager = Mock() + tm.request_metrics_exporter_manager.exporter_enabled.return_value = False + tm._lora_release_tasks = set() + if enable_lora: + tm.enable_lora = True + tm.server_args.max_loaded_loras = None + tm.lora_registry = LoRARegistry( + [LoRARef(lora_id="adapter", lora_name="adapter")] + ) + else: + tm._validate_and_resolve_lora = AsyncMock(return_value=None) return tm @@ -703,7 +723,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): tm = _make_tokenizer_manager(self) rid = "d_single" tm.rid_to_state[rid] = _make_req_state(rid) - tm._release_req_states_on_failure([rid]) + tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]}) self.assertNotIn(rid, tm.rid_to_state) def test_undelivered_batch_removes_all(self): @@ -711,7 +731,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): rids = ["d0", "d1", "d2"] for r in rids: tm.rid_to_state[r] = _make_req_state(r) - tm._release_req_states_on_failure(rids) + tm._release_req_states_on_failure(dict(tm.rid_to_state)) for r in rids: self.assertNotIn(r, tm.rid_to_state) @@ -719,7 +739,12 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): """A rid that is no longer present must not raise.""" tm = _make_tokenizer_manager(self) tm.rid_to_state["p1"] = _make_req_state("p1") - tm._release_req_states_on_failure(["p1", "already_gone"]) + tm._release_req_states_on_failure( + { + "p1": tm.rid_to_state["p1"], + "already_gone": _make_req_state("already_gone"), + } + ) self.assertNotIn("p1", tm.rid_to_state) def test_dispatched_single_is_aborted_and_state_kept(self): @@ -732,8 +757,8 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): state = _make_req_state(rid) state.dispatched = True tm.rid_to_state[rid] = state - tm._release_req_states_on_failure([rid]) - tm._release_req_states_on_failure([rid]) + tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]}) + tm._release_req_states_on_failure({rid: tm.rid_to_state[rid]}) sent = [c.args[0] for c in tm._dispatch_to_scheduler.call_args_list] self.assertEqual( @@ -753,7 +778,7 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): live.dispatched = True tm.rid_to_state[delivered] = live tm.rid_to_state[undelivered] = _make_req_state(undelivered) - tm._release_req_states_on_failure([delivered, undelivered]) + tm._release_req_states_on_failure(dict(tm.rid_to_state)) sent = [c.args[0] for c in tm._dispatch_to_scheduler.call_args_list] self.assertEqual([type(m) for m in sent], [AbortReq]) @@ -772,12 +797,56 @@ class TestReleaseReqStatesOnFailure(CustomTestCase): tm.rid_to_state[undelivered] = _make_req_state(undelivered) with self.assertLogs(level="ERROR"): - tm._release_req_states_on_failure([delivered, undelivered]) + tm._release_req_states_on_failure(dict(tm.rid_to_state)) self.assertIn(delivered, tm.rid_to_state) self.assertFalse(live.abort_sent) self.assertNotIn(undelivered, tm.rid_to_state) + def test_completed_sample_reuse_survives_sibling_cleanup(self): + """Batch cleanup must preserve a reused sample RID, adapter ref and encoder gate.""" + + async def drive(dispatched): + tm = _make_tm_for_generate(self, enable_lora=True) + tm._dispatch_to_scheduler = Mock() + samples = [ + GenerateReqInput(text="sample", lora_path="adapter") for _ in range(2) + ] + for sample in samples: + sample.normalize_batch_and_arguments() + tm._init_req_state(sample) + await tm._validate_and_resolve_lora(sample) + request_states = dict(tm.rid_to_state) + rid = samples[0].rid + await tm._handle_batch_output(_make_batch_str_output(rid)) + await asyncio.gather(*tm._lora_release_tasks) + + replacement = GenerateReqInput(text="new", rid=rid, lora_path="adapter") + replacement.normalize_batch_and_arguments() + tm._init_req_state(replacement) + await tm._validate_and_resolve_lora(replacement) + replacement_state = tm.rid_to_state[rid] + replacement_state.dispatched = dispatched + gate = asyncio.Event() + tm.encoder_dispatch_ready[rid] = gate + + tm._release_req_states_on_failure(request_states) + await asyncio.gather(*tm._lora_release_tasks) + self.assertIs(tm.rid_to_state.get(rid), replacement_state) + self.assertNotIn(samples[1].rid, tm.rid_to_state) + self.assertIs(tm.encoder_dispatch_ready.get(rid), gate) + self.assertFalse(gate.is_set()) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 1) + tm._dispatch_to_scheduler.assert_not_called() + + await tm._handle_batch_output(_make_batch_str_output(rid)) + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0) + + for dispatched in (False, True): + with self.subTest(dispatched=dispatched): + asyncio.run(drive(dispatched)) + class TestParallelStreamTaskCleanup(CustomTestCase): def test_failing_choice_cancels_and_closes_sibling_waiters(self): @@ -843,24 +912,22 @@ class TestGenerateRequestCleanupOnDispatchFailure(CustomTestCase): """ def test_single_failure_before_dispatch_cleans_up(self): - tm = _make_tm_for_generate(self) - rid = "single_overlen" - obj = _make_generate_obj(rid, is_single=True) - # Simulate over-length rejection during tokenization/validation. + """Validation failure must return the adapter reference acquired earlier.""" + tm = _make_tm_for_generate(self, enable_lora=True) + obj = GenerateReqInput(text="prompt", rid="single_overlen", lora_path="adapter") tm._tokenize_one_request = AsyncMock(side_effect=ValueError("input too long")) tm._send_one_request = Mock() async def drive(): - await tm.generate_request(obj).__anext__() + with self.assertRaisesRegex(ValueError, "input too long"): + await tm.generate_request(obj).__anext__() + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0) - with self.assertRaises(ValueError): - asyncio.run(drive()) - - # Got past _init_req_state (which created the entry) ... + asyncio.run(drive()) tm._tokenize_one_request.assert_awaited_once() tm._send_one_request.assert_not_called() - # ... and the entry was cleaned up rather than leaked. - self.assertNotIn(rid, tm.rid_to_state) + self.assertNotIn(obj.rid, tm.rid_to_state) def test_batch_failure_before_dispatch_cleans_up_all(self): tm = _make_tm_for_generate(self) @@ -998,5 +1065,131 @@ class TestDisconnectAfterDispatchAbortsRequest(CustomTestCase): self.assertIn(rid, tm.rid_to_state) +class TestLoRARequestLifecycle(CustomTestCase): + def test_abort_waiter_preserves_reused_rid(self): + """An old abort must neither erase a new request nor release its reference.""" + + async def drive(status_code, expected_exception): + tm = _make_tm_for_generate(self, enable_lora=True) + obj = GenerateReqInput(text="old", rid="reused", lora_path="adapter") + tm._tokenize_one_request = AsyncMock( + side_effect=lambda req: SimpleNamespace(rid=req.rid) + ) + dispatched = asyncio.Event() + + async def send(tokenized): + tm._mark_state_dispatched(tokenized.rid) + dispatched.set() + + tm._send_one_request = send + generator = tm.generate_request(obj) + waiter = asyncio.create_task(generator.__anext__()) + await dispatched.wait() + + abort = _make_abort_req(obj.rid) + abort.finished_reason["status_code"] = status_code + tm._handle_abort_req(abort) + # Reuse the ID before the old waiter resumes on its abort notification. + replacement = GenerateReqInput(text="new", rid=obj.rid, lora_path="adapter") + replacement.normalize_batch_and_arguments() + tm._init_req_state(replacement) + await tm._validate_and_resolve_lora(replacement) + replacement_state = tm.rid_to_state[obj.rid] + + with self.assertRaises(expected_exception) as ctx: + await waiter + if expected_exception is HTTPException: + self.assertEqual(ctx.exception.status_code, status_code) + await asyncio.gather(*tm._lora_release_tasks) + self.assertIs(tm.rid_to_state.get(obj.rid), replacement_state) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 1) + + await tm._handle_batch_output(_make_batch_str_output(obj.rid)) + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0) + await generator.aclose() + + for status_code, expected_exception in ( + (400, ValueError), + (503, HTTPException), + ): + with self.subTest(status_code=status_code): + asyncio.run(drive(status_code, expected_exception)) + + async def _drive_parallel_request(self, fail_second_warmup=False): + tm = _make_tm_for_generate(self, enable_lora=True) + obj = GenerateReqInput( + text=["first", "second"], + lora_path="adapter", + sampling_params={"n": 2, "max_new_tokens": 4}, + ) + tm._tokenize_one_request = AsyncMock( + side_effect=lambda req: SimpleNamespace( + rid=req.rid, + lora_id=req.lora_id, + input_ids=[1], + mm_inputs=None, + time_stats=tm.rid_to_state[req.rid].time_stats, + sampling_params=SimpleNamespace(max_new_tokens=4), + ) + ) + sent = asyncio.Queue() + + async def send(tokenized): + tm._mark_state_dispatched(tokenized.rid) + sent.put_nowait(tokenized) + + tm._send_one_request = send + generator = tm.generate_request(obj) + response = asyncio.create_task(generator.__anext__()) + try: + for i in range(2): + warmup = await asyncio.wait_for(sent.get(), timeout=1) + self.assertEqual(warmup.lora_id, "adapter") + self.assertEqual(warmup.sampling_params.max_new_tokens, 0) + reason = None + if fail_second_warmup and i == 1: + reason = { + "type": "abort", + "status_code": 400, + "message": "invalid grammar", + } + await tm._handle_batch_output( + _make_batch_str_output(warmup.rid, reason) + ) + + if fail_second_warmup: + with self.assertRaisesRegex(ValueError, "invalid grammar"): + await response + self.assertTrue(sent.empty()) # No samples were dispatched. + else: + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 4) + for remaining in (3, 2, 1, 0): + sample = await asyncio.wait_for(sent.get(), timeout=1) + await tm._handle_batch_output(_make_batch_str_output(sample.rid)) + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual( + tm.lora_registry._counters["adapter"].value(), remaining + ) + self.assertEqual(len(await response), 4) + + await asyncio.gather(*tm._lora_release_tasks) + self.assertEqual(tm.lora_registry._counters["adapter"].value(), 0) + self.assertFalse(tm.rid_to_state) + finally: + response.cancel() + await asyncio.gather(response, return_exceptions=True) + await generator.aclose() + + def test_parallel_warmups_do_not_release_sample_references(self): + """Prefix warmups must not let an adapter unload while samples use it.""" + asyncio.run(self._drive_parallel_request()) + + def test_second_parallel_warmup_failure_releases_all_references(self): + """A failed warmup must return references even after earlier warmups finish.""" + asyncio.run(self._drive_parallel_request(fail_second_warmup=True)) + + if __name__ == "__main__": unittest.main(verbosity=2)