From 6c2a759a04232ef4cb0c845528d75516ddbd9fe2 Mon Sep 17 00:00:00 2001 From: maocheng23 <35615230+maocheng23@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:32:56 -0700 Subject: [PATCH] [fix] Fix writer lock deadlock in update_weights_from_ipc during pause_generation (#22290) Co-authored-by: Claude Opus 4.6 (1M context) --- .../srt/managers/tokenizer_communicator_mixin.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_communicator_mixin.py b/python/sglang/srt/managers/tokenizer_communicator_mixin.py index 8ba0fe077..76d540168 100644 --- a/python/sglang/srt/managers/tokenizer_communicator_mixin.py +++ b/python/sglang/srt/managers/tokenizer_communicator_mixin.py @@ -712,8 +712,18 @@ class TokenizerCommunicatorMixin: self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from IPC" logger.info("Starting IPC weight update") - # This means that weight sync cannot run while requests are in progress. - async with self.model_update_lock.writer_lock: + + # Skip the writer lock when paused: readers are blocked on + # is_pause_cond so no concurrent inference can race, and + # waiting for the writer lock would deadlock because existing + # readers are stuck waiting on the paused scheduler. + async with self.is_pause_cond: + is_paused = self.is_pause + + lock_context = ( + self.model_update_lock.writer_lock if not is_paused else nullcontext() + ) + async with lock_context: result = (await self.update_weights_from_ipc_communicator(obj))[0] success, message = result.success, result.message except Exception as e: