From 3e13f1b2c725d773b4dd845d12bd82903a7dba3f Mon Sep 17 00:00:00 2001 From: HEJIAN SANG Date: Thu, 8 Oct 2026 20:17:25 -0700 Subject: [PATCH] [Perf] Serialize OpenAI chat/completion responses in one pass (#42889) Co-authored-by: Claude Opus 5.5 --- python/sglang/srt/entrypoints/http_server.py | 19 ++-- python/sglang/srt/utils/json_response.py | 11 +++ .../unit/utils/test_json_response.py | 89 +++++++++++++++++++ 3 files changed, 113 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index b659c9add8..ea9addb0ba 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -189,6 +189,7 @@ from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth from sglang.srt.utils.json_response import ( SGLangORJSONResponse, dumps_json, + model_json_response, orjson_response, ) from sglang.srt.utils.msgspec_utils import msgspec_to_builtins @@ -1778,8 +1779,10 @@ async def continue_generation( @app.post("/v1/completions", dependencies=[Depends(validate_json_request)]) async def openai_v1_completions(request: CompletionRequest, raw_request: Request): """OpenAI-compatible text completion endpoint.""" - return await raw_request.app.state.openai_serving_completion.handle_request( - request, raw_request + return model_json_response( + await raw_request.app.state.openai_serving_completion.handle_request( + request, raw_request + ) ) @@ -1788,8 +1791,10 @@ async def openai_v1_chat_completions( request: ChatCompletionRequest, raw_request: Request ): """OpenAI-compatible chat completion endpoint.""" - return await raw_request.app.state.openai_serving_chat.handle_request( - request, raw_request + return model_json_response( + await raw_request.app.state.openai_serving_chat.handle_request( + request, raw_request + ) ) @@ -2112,8 +2117,10 @@ async def sagemaker_chat_completions( request: ChatCompletionRequest, raw_request: Request ): """OpenAI-compatible chat completion endpoint.""" - return await raw_request.app.state.openai_serving_chat.handle_request( - request, raw_request + return model_json_response( + await raw_request.app.state.openai_serving_chat.handle_request( + request, raw_request + ) ) diff --git a/python/sglang/srt/utils/json_response.py b/python/sglang/srt/utils/json_response.py index aa13dfb2cf..398fdc9652 100644 --- a/python/sglang/srt/utils/json_response.py +++ b/python/sglang/srt/utils/json_response.py @@ -4,6 +4,7 @@ from typing import Any import orjson from fastapi.responses import Response +from pydantic import BaseModel # Keep response serialization behavior consistent across endpoints: # - Support non-string dictionary keys used in some metadata payloads. @@ -28,3 +29,13 @@ class SGLangORJSONResponse(Response): def orjson_response(content: Any, status_code: int = 200) -> Response: """Create a JSON response with stable ORJSON serialization options.""" return SGLangORJSONResponse(content=content, status_code=status_code) + + +def model_json_response(content: Any) -> Any: + """Serialize a pydantic result in one pass, skipping FastAPI's jsonable_encoder walk.""" + if not isinstance(content, BaseModel): + return content + return Response( + content=content.model_dump_json(by_alias=True), + media_type="application/json", + ) diff --git a/test/registered/unit/utils/test_json_response.py b/test/registered/unit/utils/test_json_response.py index 70aac292ce..25f93c25b8 100644 --- a/test/registered/unit/utils/test_json_response.py +++ b/test/registered/unit/utils/test_json_response.py @@ -2,10 +2,23 @@ import unittest import numpy as np import orjson +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatCompletionTokenLogprob, + ChatMessage, + ChoiceLogprobs, + JsonSchemaResponseFormat, + TopLogprob, + UsageInfo, +) from sglang.srt.utils.json_response import ( SGLangORJSONResponse, dumps_json, + model_json_response, orjson_response, ) from sglang.test.ci.ci_register import register_cpu_ci @@ -52,5 +65,81 @@ class TestJSONResponseUtils(unittest.TestCase): self.assertIsNone(parsed["value"]) +def _chat_response_with_top_logprobs(logprob: float) -> ChatCompletionResponse: + top = [ + TopLogprob(token="é", bytes=[195, 169], logprob=logprob), + TopLogprob(token=" 中", bytes=[32, 228, 184, 173], logprob=-1e-5), + ] + return ChatCompletionResponse( + id="req-0", + created=0, + model="m", + choices=[ + ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", content="é 中"), + logprobs=ChoiceLogprobs( + content=[ + ChatCompletionTokenLogprob( + token="é", + bytes=[195, 169], + logprob=-0.25, + top_logprobs=top, + ) + ] + ), + finish_reason="length", + meta_info={ + "output_token_logprobs": [(-0.25, 7, "é")], + "output_top_logprobs": [[(logprob, 7, "é"), (-1e16, 8, None)]], + }, + ) + ], + usage=UsageInfo(prompt_tokens=3, completion_tokens=1, total_tokens=4), + ) + + +class TestModelJSONResponse(unittest.TestCase): + def test_matches_fastapi_encoding(self): + response = _chat_response_with_top_logprobs(-3.5) + expected = JSONResponse(jsonable_encoder(response)).body + + rendered = model_json_response(response) + + self.assertEqual(rendered.status_code, 200) + self.assertEqual(rendered.media_type, "application/json") + self.assertEqual(orjson.loads(rendered.body), orjson.loads(expected)) + + def test_keeps_the_response_model_serializer(self): + """The response's wrap serializer must still drop an unset `sglext`.""" + rendered = model_json_response(_chat_response_with_top_logprobs(-3.5)) + + self.assertNotIn("sglext", orjson.loads(rendered.body)) + + def test_applies_field_aliases_like_fastapi(self): + schema = JsonSchemaResponseFormat(name="s", schema={"type": "object"}) + + parsed = orjson.loads(model_json_response(schema).body) + + self.assertEqual(parsed, jsonable_encoder(schema)) + self.assertEqual(parsed["schema"], {"type": "object"}) + + def test_maps_non_finite_logprobs_to_null(self): + response = _chat_response_with_top_logprobs(float("-inf")) + + parsed = orjson.loads(model_json_response(response).body) + + choice = parsed["choices"][0] + self.assertIsNone( + choice["logprobs"]["content"][0]["top_logprobs"][0]["logprob"] + ) + self.assertIsNone(choice["meta_info"]["output_top_logprobs"][0][0][0]) + + def test_passes_non_models_through(self): + error = orjson_response({"error": "bad"}, status_code=400) + + self.assertIs(model_json_response(error), error) + + if __name__ == "__main__": unittest.main()