Fix broken streaming response with --incremental-streaming-output (#22549)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-12 15:05:58 -07:00
parent 701a0e0c25
commit 1f8df97054
2 changed files with 93 additions and 1 deletions
@@ -738,7 +738,11 @@ class OpenAIServingChat(OpenAIServingBase):
                    stream_started = True

                stream_buffer = stream_buffers.get(index, "")
-                delta = content["text"][len(stream_buffer) :]
+                if self.tokenizer_manager.server_args.incremental_streaming_output:
+                    # content["text"] is already the incremental delta
+                    delta = content["text"]
+                else:
+                    delta = content["text"][len(stream_buffer) :]
                stream_buffers[index] = stream_buffer + delta

                # Handle reasoning content