fix(openai): keep chat stream created timestamp stable (#43026)
Co-authored-by: ai-jz <ai-jz@users.noreply.github.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
co-authored by
ai-jz
Xinyuan Tong
parent
4c5ed0597e
commit
3f1e185f6e
@@ -887,6 +887,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
content: dict[str, Any],
|
||||
index: int,
|
||||
request: ChatCompletionRequest,
|
||||
created: int,
|
||||
stream_offsets: dict[int, int],
|
||||
reasoning_parser_dict: dict,
|
||||
parser_dict: dict,
|
||||
@@ -933,7 +934,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
yield build_sse_content(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=index,
|
||||
reasoning_content=reasoning_text,
|
||||
@@ -951,7 +952,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
content,
|
||||
request,
|
||||
has_tool_calls,
|
||||
continuous_usage_stats,
|
||||
created=created,
|
||||
continuous_usage_stats=continuous_usage_stats,
|
||||
flush=finish_reason_type is not None and finish_reason_type != "abort",
|
||||
):
|
||||
if chunk:
|
||||
@@ -961,7 +963,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if finish_reason_type is not None and index in parser_dict:
|
||||
parser = parser_dict[index]
|
||||
remaining_chunk = self._check_for_unstreamed_tool_args(
|
||||
parser, content, request, index
|
||||
parser, content, request, index, created=created
|
||||
)
|
||||
if remaining_chunk:
|
||||
yield remaining_chunk
|
||||
@@ -980,7 +982,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
yield build_sse_content(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=index,
|
||||
content=delta,
|
||||
@@ -1008,7 +1010,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
yield build_sse_content(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=index,
|
||||
logprobs=remaining_logprobs,
|
||||
@@ -1958,6 +1960,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
raw_request: Request,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Generate streaming chat completion response"""
|
||||
created = int(time.time())
|
||||
|
||||
# Parsers for tool calls and reasoning
|
||||
parser_dict = {}
|
||||
reasoning_parser_dict = {}
|
||||
@@ -2095,7 +2099,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
# K2 templates require preserved thinking, including empty reasoning.
|
||||
yield build_sse_content(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=index,
|
||||
role="assistant",
|
||||
@@ -2114,6 +2118,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
content=content,
|
||||
index=index,
|
||||
request=request,
|
||||
created=created,
|
||||
stream_offsets=stream_offsets,
|
||||
reasoning_parser_dict=reasoning_parser_dict,
|
||||
parser_dict=parser_dict,
|
||||
@@ -2139,7 +2144,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
matched_stop = finish_reason_data.get("matched")
|
||||
yield build_sse_content(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=idx,
|
||||
finish_reason=final_finish_reason,
|
||||
@@ -2155,7 +2160,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
hidden_states_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[
|
||||
ChatCompletionResponseStreamChoice(
|
||||
index=index,
|
||||
@@ -2223,7 +2228,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if sglext_non_ids is not None:
|
||||
sglext_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[],
|
||||
model=request.model,
|
||||
sglext=sglext_non_ids,
|
||||
@@ -2232,7 +2237,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if sglext_ids is not None:
|
||||
sglext_ids_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[],
|
||||
model=request.model,
|
||||
sglext=sglext_ids,
|
||||
@@ -2241,7 +2246,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
elif sglext_non_ids is not None or sglext_ids is not None:
|
||||
sglext_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[], # sglext is at response level
|
||||
model=request.model,
|
||||
sglext=sglext_full,
|
||||
@@ -2274,7 +2279,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
usage_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[], # Empty choices array as per OpenAI spec
|
||||
model=request.model,
|
||||
usage=usage,
|
||||
@@ -3058,6 +3063,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
content: dict[str, Any],
|
||||
request: ChatCompletionRequest,
|
||||
has_tool_calls: dict[int, bool],
|
||||
created: int,
|
||||
continuous_usage_stats: bool = False,
|
||||
flush: bool = False,
|
||||
):
|
||||
@@ -3119,7 +3125,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[choice_data],
|
||||
model=request.model,
|
||||
)
|
||||
@@ -3172,7 +3178,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[choice_data],
|
||||
model=request.model,
|
||||
)
|
||||
@@ -3197,6 +3203,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
content: dict[str, Any],
|
||||
request: ChatCompletionRequest,
|
||||
index: int,
|
||||
created: int,
|
||||
) -> str | None:
|
||||
"""
|
||||
Check for any remaining tool call arguments that need to be streamed
|
||||
@@ -3258,7 +3265,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=int(time.time()),
|
||||
created=created,
|
||||
choices=[choice_data],
|
||||
model=request.model,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user