Fix TokenizerManager bottleneck for offline generation (#16456)

python/sglang/srt/environ.py

@@ -419,6 +419,9 @@ class Envs:
     # Tokenizer
     SGLANG_PATCH_TOKENIZER = EnvBool(False)  # TODO enable by default
 
+    # TokenizerManager
+    SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4)
+
     # fmt: on
 
 

python/sglang/srt/managers/tokenizer_manager.py

@@ -116,6 +116,8 @@ from sglang.utils import TypeBasedDispatcher, get_exception_traceback
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
+_REQUEST_STATE_WAIT_TIMEOUT = envs.SGLANG_REQUEST_STATE_WAIT_TIMEOUT.get()
+
 logger = logging.getLogger(__name__)
 
 
@@ -1081,7 +1083,9 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
         is_stream = getattr(obj, "stream", False)
         while True:
             try:
-                await asyncio.wait_for(state.event.wait(), timeout=4)
+                await asyncio.wait_for(
+                    state.event.wait(), timeout=_REQUEST_STATE_WAIT_TIMEOUT
+                )
             except asyncio.TimeoutError:
                 if (
                     request is not None