[Perf] Serialize OpenAI chat/completion responses in one pass (#42889)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
HEJIAN SANG
2026-10-09 11:17:25 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 438df9a2a4
commit 3e13f1b2c7
3 changed files with 113 additions and 6 deletions
+13 -6
View File
@@ -189,6 +189,7 @@ from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth
from sglang.srt.utils.json_response import (
SGLangORJSONResponse,
dumps_json,
model_json_response,
orjson_response,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
@@ -1778,8 +1779,10 @@ async def continue_generation(
@app.post("/v1/completions", dependencies=[Depends(validate_json_request)])
async def openai_v1_completions(request: CompletionRequest, raw_request: Request):
"""OpenAI-compatible text completion endpoint."""
return await raw_request.app.state.openai_serving_completion.handle_request(
request, raw_request
return model_json_response(
await raw_request.app.state.openai_serving_completion.handle_request(
request, raw_request
)
)
@@ -1788,8 +1791,10 @@ async def openai_v1_chat_completions(
request: ChatCompletionRequest, raw_request: Request
):
"""OpenAI-compatible chat completion endpoint."""
return await raw_request.app.state.openai_serving_chat.handle_request(
request, raw_request
return model_json_response(
await raw_request.app.state.openai_serving_chat.handle_request(
request, raw_request
)
)
@@ -2112,8 +2117,10 @@ async def sagemaker_chat_completions(
request: ChatCompletionRequest, raw_request: Request
):
"""OpenAI-compatible chat completion endpoint."""
return await raw_request.app.state.openai_serving_chat.handle_request(
request, raw_request
return model_json_response(
await raw_request.app.state.openai_serving_chat.handle_request(
request, raw_request
)
)
+11
View File
@@ -4,6 +4,7 @@ from typing import Any
import orjson
from fastapi.responses import Response
from pydantic import BaseModel
# Keep response serialization behavior consistent across endpoints:
# - Support non-string dictionary keys used in some metadata payloads.
@@ -28,3 +29,13 @@ class SGLangORJSONResponse(Response):
def orjson_response(content: Any, status_code: int = 200) -> Response:
"""Create a JSON response with stable ORJSON serialization options."""
return SGLangORJSONResponse(content=content, status_code=status_code)
def model_json_response(content: Any) -> Any:
"""Serialize a pydantic result in one pass, skipping FastAPI's jsonable_encoder walk."""
if not isinstance(content, BaseModel):
return content
return Response(
content=content.model_dump_json(by_alias=True),
media_type="application/json",
)