Fix streaming token ids data loss under load (#19977)

Signed-off-by: Vladislav Nosivskoy <vladnosiv@gmail.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>

python/sglang/srt/managers/tokenizer_manager.py

@@ -1147,10 +1147,26 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
                     )
                 continue
 
-            out = state.out_list[-1]
+            # Drain all pending outputs atomically. For streaming, every
-
+            # chunk must be yielded to avoid dropping token deltas. For
+            # non-streaming only the latest cumulative output matters.
+            pending = state.out_list if is_stream else state.out_list[-1:]
             state.out_list = []
-            if state.finished:
+            finished = state.finished
+            state.event.clear()
+
+            if is_stream and len(pending) > 1:
+                logger.warning(
+                    "Streaming backlog: rid=%s, draining %d queued chunks. "
+                    "This may inflate P99 TBT for affected requests.",
+                    obj.rid,
+                    len(pending),
+                )
+
+            for i, out in enumerate(pending):
+                is_last = i == len(pending) - 1
+
+                if finished and is_last:
                     # For non-streaming cases, response has not been sent yet (`response_sent_to_client_time` has not been set yet).
                     # Record response sent time right before we log finished results and metrics.
                     if not state.time_stats.response_sent_to_client_time:
@@ -1176,7 +1192,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
                         finish_reason = out["meta_info"]["finish_reason"]
                         if (
                             finish_reason.get("type") == "abort"
-                        and finish_reason.get("status_code") == HTTPStatus.BAD_REQUEST
+                            and finish_reason.get("status_code")
+                            == HTTPStatus.BAD_REQUEST
                         ):
                             if not is_stream:
                                 raise ValueError(finish_reason["message"])
@@ -1210,8 +1227,6 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
                     yield out
                     break
 
-            state.event.clear()
-
                 if is_stream:
                     # Record response sent time right before we send response.
                     if not state.time_stats.response_sent_to_client_time:
@@ -1220,7 +1235,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
                             "response_sent_to_client_ts"
                         ] = state.time_stats.get_response_sent_to_client_realtime()
                     yield out
-            else:
+
+            if finished:
+                break
+
+            if not is_stream:
                 if (
                     request is not None
                     and not obj.background