Fix flaky streaming logprobs test by handling detokenizer text buffering (#17687)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/entrypoints/openai/serving_chat.py

@@ -627,16 +627,22 @@ class OpenAIServingChat(OpenAIServingBase):
                 routed_experts[index] = content["meta_info"].get("routed_experts", None)
 
                 # Handle logprobs
+                finish_reason = content["meta_info"]["finish_reason"]
                 choice_logprobs = None
                 if request.logprobs:
-                    choice_logprobs = self._process_streaming_logprobs(
+                    n_prev_token = n_prev_tokens.get(index, 0)
-                        content, n_prev_tokens.get(index, 0)
+                    total_output_logprobs = len(
-                    )
-                    n_prev_tokens[index] = len(
                         content["meta_info"]["output_token_logprobs"]
                     )
-
+                    # When finish_reason is set and all logprobs have been sent,
-                finish_reason = content["meta_info"]["finish_reason"]
+                    # any remaining text is just buffered text being flushed by the
+                    # detokenizer (it holds back text at word boundaries). Return None
+                    # for logprobs since no new tokens were generated for this text.
+                    if n_prev_token < total_output_logprobs or finish_reason is None:
+                        choice_logprobs = self._process_streaming_logprobs(
+                            content, n_prev_token
+                        )
+                    n_prev_tokens[index] = total_output_logprobs
                 finish_reason_type = finish_reason["type"] if finish_reason else None
 
                 # Track finish_reason for each index

python/sglang/srt/entrypoints/openai/serving_completions.py

@@ -242,19 +242,34 @@ class OpenAIServingCompletion(OpenAIServingBase):
                         input_top_logprobs = None
 
                     n_prev_token = n_prev_tokens.get(index, 0)
+                    total_output_logprobs = len(
+                        content["meta_info"]["output_token_logprobs"]
+                    )
+                    output_logprobs_slice = content["meta_info"][
+                        "output_token_logprobs"
+                    ][n_prev_token:]
+                    finish_reason_for_logprobs = content["meta_info"]["finish_reason"]
+
+                    # When finish_reason is set and all logprobs have been sent,
+                    # any remaining text is just buffered text being flushed by the
+                    # detokenizer (it holds back text at word boundaries). Return None
+                    # for logprobs since no new tokens were generated for this text.
+                    if (
+                        len(output_logprobs_slice) == 0
+                        and finish_reason_for_logprobs is not None
+                        and input_token_logprobs is None
+                    ):
+                        logprobs = None
+                    else:
                         logprobs = to_openai_style_logprobs(
                             input_token_logprobs=input_token_logprobs,
                             input_top_logprobs=input_top_logprobs,
-                        output_token_logprobs=content["meta_info"][
+                            output_token_logprobs=output_logprobs_slice,
-                            "output_token_logprobs"
-                        ][n_prev_token:],
                             output_top_logprobs=content["meta_info"].get(
                                 "output_top_logprobs", []
                             )[n_prev_token:],
                         )
-                    n_prev_tokens[index] = len(
+                    n_prev_tokens[index] = total_output_logprobs
-                        content["meta_info"]["output_token_logprobs"]
-                    )
 
                 # Generate delta
                 delta = text[len(stream_buffer) :]

test/registered/openai_server/basic/test_openai_server.py

@@ -156,10 +156,10 @@ class TestOpenAIServer(CustomTestCase):
             is_first = is_firsts.get(index, True)
 
             if logprobs:
-                assert response.choices[0].logprobs, f"no logprobs in response"
+                # When finish_reason is set, logprobs may be None if this chunk
-                print(
+                # only contains buffered text being flushed (no new tokens generated).
-                    f"{response=}, {response.choices[0]=}, {response.choices[0].logprobs=}"
+                # The detokenizer holds back text at word boundaries during streaming.
-                )
+                if response.choices[0].logprobs is not None:
                     assert isinstance(
                         response.choices[0].logprobs.tokens[0], str
                     ), f"{response.choices[0].logprobs.tokens[0]} is not a string"