[Perf] Serialize OpenAI chat/completion responses in one pass (#42889)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
438df9a2a4
commit
3e13f1b2c7
@@ -189,6 +189,7 @@ from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth
|
||||
from sglang.srt.utils.json_response import (
|
||||
SGLangORJSONResponse,
|
||||
dumps_json,
|
||||
model_json_response,
|
||||
orjson_response,
|
||||
)
|
||||
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
|
||||
@@ -1778,8 +1779,10 @@ async def continue_generation(
|
||||
@app.post("/v1/completions", dependencies=[Depends(validate_json_request)])
|
||||
async def openai_v1_completions(request: CompletionRequest, raw_request: Request):
|
||||
"""OpenAI-compatible text completion endpoint."""
|
||||
return await raw_request.app.state.openai_serving_completion.handle_request(
|
||||
request, raw_request
|
||||
return model_json_response(
|
||||
await raw_request.app.state.openai_serving_completion.handle_request(
|
||||
request, raw_request
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1788,8 +1791,10 @@ async def openai_v1_chat_completions(
|
||||
request: ChatCompletionRequest, raw_request: Request
|
||||
):
|
||||
"""OpenAI-compatible chat completion endpoint."""
|
||||
return await raw_request.app.state.openai_serving_chat.handle_request(
|
||||
request, raw_request
|
||||
return model_json_response(
|
||||
await raw_request.app.state.openai_serving_chat.handle_request(
|
||||
request, raw_request
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -2112,8 +2117,10 @@ async def sagemaker_chat_completions(
|
||||
request: ChatCompletionRequest, raw_request: Request
|
||||
):
|
||||
"""OpenAI-compatible chat completion endpoint."""
|
||||
return await raw_request.app.state.openai_serving_chat.handle_request(
|
||||
request, raw_request
|
||||
return model_json_response(
|
||||
await raw_request.app.state.openai_serving_chat.handle_request(
|
||||
request, raw_request
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
import orjson
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Keep response serialization behavior consistent across endpoints:
|
||||
# - Support non-string dictionary keys used in some metadata payloads.
|
||||
@@ -28,3 +29,13 @@ class SGLangORJSONResponse(Response):
|
||||
def orjson_response(content: Any, status_code: int = 200) -> Response:
|
||||
"""Create a JSON response with stable ORJSON serialization options."""
|
||||
return SGLangORJSONResponse(content=content, status_code=status_code)
|
||||
|
||||
|
||||
def model_json_response(content: Any) -> Any:
|
||||
"""Serialize a pydantic result in one pass, skipping FastAPI's jsonable_encoder walk."""
|
||||
if not isinstance(content, BaseModel):
|
||||
return content
|
||||
return Response(
|
||||
content=content.model_dump_json(by_alias=True),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user