From 3f1e185f6e83ca02ec19498aa2917ccaa0f0fd43 Mon Sep 17 00:00:00 2001 From: ai-jz <156989844+ai-jz@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:43:33 -0700 Subject: [PATCH] fix(openai): keep chat stream created timestamp stable (#43026) Co-authored-by: ai-jz Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> --- .../srt/entrypoints/openai/serving_chat.py | 37 ++++++++++-------- .../entrypoints/openai/test_serving_chat.py | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index d3b15a4205..20328541e0 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -887,6 +887,7 @@ class OpenAIServingChat(OpenAIServingBase): content: dict[str, Any], index: int, request: ChatCompletionRequest, + created: int, stream_offsets: dict[int, int], reasoning_parser_dict: dict, parser_dict: dict, @@ -933,7 +934,7 @@ class OpenAIServingChat(OpenAIServingBase): yield build_sse_content( chunk_id=content["meta_info"]["id"], - created=int(time.time()), + created=created, model=request.model, index=index, reasoning_content=reasoning_text, @@ -951,7 +952,8 @@ class OpenAIServingChat(OpenAIServingBase): content, request, has_tool_calls, - continuous_usage_stats, + created=created, + continuous_usage_stats=continuous_usage_stats, flush=finish_reason_type is not None and finish_reason_type != "abort", ): if chunk: @@ -961,7 +963,7 @@ class OpenAIServingChat(OpenAIServingBase): if finish_reason_type is not None and index in parser_dict: parser = parser_dict[index] remaining_chunk = self._check_for_unstreamed_tool_args( - parser, content, request, index + parser, content, request, index, created=created ) if remaining_chunk: yield remaining_chunk @@ -980,7 +982,7 @@ class OpenAIServingChat(OpenAIServingBase): yield build_sse_content( chunk_id=content["meta_info"]["id"], - created=int(time.time()), + created=created, model=request.model, index=index, content=delta, @@ -1008,7 +1010,7 @@ class OpenAIServingChat(OpenAIServingBase): yield build_sse_content( chunk_id=content["meta_info"]["id"], - created=int(time.time()), + created=created, model=request.model, index=index, logprobs=remaining_logprobs, @@ -1958,6 +1960,8 @@ class OpenAIServingChat(OpenAIServingBase): raw_request: Request, ) -> AsyncGenerator[str, None]: """Generate streaming chat completion response""" + created = int(time.time()) + # Parsers for tool calls and reasoning parser_dict = {} reasoning_parser_dict = {} @@ -2095,7 +2099,7 @@ class OpenAIServingChat(OpenAIServingBase): # K2 templates require preserved thinking, including empty reasoning. yield build_sse_content( chunk_id=content["meta_info"]["id"], - created=int(time.time()), + created=created, model=request.model, index=index, role="assistant", @@ -2114,6 +2118,7 @@ class OpenAIServingChat(OpenAIServingBase): content=content, index=index, request=request, + created=created, stream_offsets=stream_offsets, reasoning_parser_dict=reasoning_parser_dict, parser_dict=parser_dict, @@ -2139,7 +2144,7 @@ class OpenAIServingChat(OpenAIServingBase): matched_stop = finish_reason_data.get("matched") yield build_sse_content( chunk_id=content["meta_info"]["id"], - created=int(time.time()), + created=created, model=request.model, index=idx, finish_reason=final_finish_reason, @@ -2155,7 +2160,7 @@ class OpenAIServingChat(OpenAIServingBase): ) hidden_states_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[ ChatCompletionResponseStreamChoice( index=index, @@ -2223,7 +2228,7 @@ class OpenAIServingChat(OpenAIServingBase): if sglext_non_ids is not None: sglext_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[], model=request.model, sglext=sglext_non_ids, @@ -2232,7 +2237,7 @@ class OpenAIServingChat(OpenAIServingBase): if sglext_ids is not None: sglext_ids_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[], model=request.model, sglext=sglext_ids, @@ -2241,7 +2246,7 @@ class OpenAIServingChat(OpenAIServingBase): elif sglext_non_ids is not None or sglext_ids is not None: sglext_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[], # sglext is at response level model=request.model, sglext=sglext_full, @@ -2274,7 +2279,7 @@ class OpenAIServingChat(OpenAIServingBase): ) usage_chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[], # Empty choices array as per OpenAI spec model=request.model, usage=usage, @@ -3058,6 +3063,7 @@ class OpenAIServingChat(OpenAIServingBase): content: dict[str, Any], request: ChatCompletionRequest, has_tool_calls: dict[int, bool], + created: int, continuous_usage_stats: bool = False, flush: bool = False, ): @@ -3119,7 +3125,7 @@ class OpenAIServingChat(OpenAIServingBase): ) chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[choice_data], model=request.model, ) @@ -3172,7 +3178,7 @@ class OpenAIServingChat(OpenAIServingBase): ) chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[choice_data], model=request.model, ) @@ -3197,6 +3203,7 @@ class OpenAIServingChat(OpenAIServingBase): content: dict[str, Any], request: ChatCompletionRequest, index: int, + created: int, ) -> str | None: """ Check for any remaining tool call arguments that need to be streamed @@ -3258,7 +3265,7 @@ class OpenAIServingChat(OpenAIServingBase): chunk = ChatCompletionStreamResponse( id=content["meta_info"]["id"], - created=int(time.time()), + created=created, choices=[choice_data], model=request.model, ) diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 274f03b4d3..27eb55fbe8 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -1871,6 +1871,7 @@ class ServingChatTestCase(CustomTestCase): content=content, request=request, index=0, + created=123, ) # Should return a chunk with remaining arguments @@ -1879,6 +1880,7 @@ class ServingChatTestCase(CustomTestCase): # Parse the result to verify content self.assertTrue(result.startswith("data: ")) chunk = json.loads(result[6:]) + self.assertEqual(chunk["created"], 123) tool_calls = chunk["choices"][0]["delta"]["tool_calls"] self.assertEqual(len(tool_calls), 1) arguments = tool_calls[0]["function"]["arguments"] @@ -1924,6 +1926,7 @@ class ServingChatTestCase(CustomTestCase): content=content, request=request, index=0, + created=123, ) # Should return None since no completion is needed @@ -1962,6 +1965,7 @@ class ServingChatTestCase(CustomTestCase): content=content, request=request, index=0, + created=123, ) self.assertIsNone(result, "Should not append encoded quotes") @@ -1997,6 +2001,7 @@ class ServingChatTestCase(CustomTestCase): content=content, request=request, index=0, + created=123, ) self.assertIsNotNone(result, "Should return chunk with remaining arguments") @@ -2032,6 +2037,7 @@ class ServingChatTestCase(CustomTestCase): content=content, request=request, index=0, + created=123, ) # Should return None since there's no parser data @@ -2563,6 +2569,7 @@ class ServingChatTestCase(CustomTestCase): content={"meta_info": {"id": "chatcmpl-test"}}, request=req, has_tool_calls={}, + created=123, ) # Get first yielded SSE line line = None @@ -2577,6 +2584,7 @@ class ServingChatTestCase(CustomTestCase): self.assertTrue(line.startswith("data: ")) payload = json.loads(line[len("data: ") :]) + self.assertEqual(payload["created"], 123) tool_calls = payload["choices"][0]["delta"]["tool_calls"] self.assertEqual(tool_calls[0]["id"], "functions.get_weather:1") @@ -3186,12 +3194,41 @@ class ServingChatTestCase(CustomTestCase): parsed.append(json.loads(c[len("data: ") :])) return parsed + def test_streaming_created_is_stable_across_chunks(self): + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + stream=True, + stream_options={"include_usage": True}, + ) + + with patch( + "sglang.srt.entrypoints.openai.serving_chat.time.time", + side_effect=[100, 101, 102, 103], + ) as current_time: + chunks = self._run_chat_stream(Mock(), req) + + parsed = self._parse_chunks(chunks) + self.assertGreater(len(parsed), 1) + identities = { + (chunk["id"], chunk["created"], chunk["model"]) for chunk in parsed + } + self.assertEqual(len(identities), 1) + response_id, created, model = identities.pop() + self.assertTrue(response_id) + self.assertEqual((created, model), (100, "x")) + self.assertTrue( + any(not chunk["choices"] and chunk.get("usage") for chunk in parsed) + ) + current_time.assert_called_once_with() + async def _collect_stream_content(self, content, choice_logprobs, req): chunks = [] async for chunk in self.chat._generate_stream_content( content=content, index=0, request=req, + created=123, stream_offsets={}, reasoning_parser_dict={}, parser_dict={}, @@ -4066,6 +4103,7 @@ class ServingChatTestCase(CustomTestCase): content=content, index=0, request=req, + created=123, stream_offsets={}, reasoning_parser_dict={}, parser_dict={},