[Fix] Preserve HTTP errors for non-streaming /v1/messages (#43796)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Vegetog
2026-10-13 10:10:45 +08:00
committed by GitHub
co-authored by Xinyuan Tong Xinyuan Tong
parent 45418d7dc6
commit 75a0d91c1f
2 changed files with 100 additions and 21 deletions
@@ -12,7 +12,7 @@ import logging
import uuid
from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Union
from fastapi import Request
from fastapi import HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel, ValidationError
@@ -757,6 +757,15 @@ class AnthropicServing:
)
except asyncio.CancelledError:
raise
except HTTPException as e:
logger.warning(
"Non-streaming request rejected: %s %s", e.status_code, e.detail
)
return self._error_response(
status_code=e.status_code,
error_type=ERROR_TYPE_MAP.get(e.status_code, "api_error"),
message=_scrub_error_message(str(e.detail), e.status_code),
)
except Exception as e:
logger.exception("Error processing Anthropic request: %s", e)
return self._error_response(
@@ -1,12 +1,14 @@
import asyncio
import json
import unittest
from http import HTTPStatus
from types import SimpleNamespace
from sglang.test.test_utils import maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel
from fastapi import HTTPException # noqa: E402
from fastapi.responses import JSONResponse # noqa: E402
from jinja2 import Environment # noqa: E402
@@ -56,14 +58,15 @@ class _FakeOpenAIServingChat:
class _FakeNonStreamingErrorOpenAI:
"""Returns a configurable error response from the OpenAI handler."""
"""Returns an error response or raises an exception from the OpenAI handler."""
supports_inline_system = False
def __init__(self, status_code=400, body=None, content=None):
def __init__(self, status_code=400, body=None, content=None, exception=None):
self._status_code = status_code
self._body = body
self._content = content
self._exception = exception
def _validate_request(self, chat_request):
return None
@@ -74,6 +77,8 @@ class _FakeNonStreamingErrorOpenAI:
async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
if self._exception is not None:
raise self._exception
if self._body is not None:
# Build a response object exposing raw bytes via `.body`.
return SimpleNamespace(status_code=self._status_code, body=self._body)
@@ -89,24 +94,6 @@ class _FakeNonStreamingErrorOpenAI:
)
class _FakeNonStreamingOpenAI:
"""Returns a configurable ChatCompletionResponse from the OpenAI handler."""
def __init__(self, response):
self._response = response
def _validate_request(self, chat_request):
return None
def _convert_to_internal_request(self, chat_request, raw_request):
return SimpleNamespace(), chat_request
async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
return self._response
def _chunk(choices=None, usage=None):
data = {
"id": "chatcmpl-test",
@@ -179,6 +166,12 @@ class TestAnthropicServing(unittest.TestCase):
data.update(overrides)
return AnthropicMessagesRequest.model_validate(data)
def _non_streaming_exception_response(self, exception):
serving = AnthropicServing(_FakeNonStreamingErrorOpenAI(exception=exception))
return asyncio.run(
serving.handle_messages(self._anthropic_request(stream=False), object())
)
def _tool_result_request(self, content, tools=None):
overrides = {
"stream": False,
@@ -551,6 +544,83 @@ class TestAnthropicServing(unittest.TestCase):
self.assertEqual(payload["error"]["type"], "invalid_request_error")
self.assertEqual(payload["error"]["message"], "context length exceeded")
def test_non_streaming_queue_full_http_exception_preserves_503(self):
for status_code in (503, HTTPStatus.SERVICE_UNAVAILABLE):
with self.subTest(status_code=status_code, code_type=type(status_code)):
response = self._non_streaming_exception_response(
HTTPException(
status_code=status_code,
detail="The request queue is full: sensitive internals /opt/secret",
)
)
self.assertEqual(response.status_code, 503)
self.assertEqual(
json.loads(response.body),
{
"type": "error",
"error": {
"type": "overloaded_error",
"message": "Internal server error",
},
},
)
def test_non_streaming_http_exception_4xx_preserves_status_and_message(self):
for status_code, expected_type in [
(400, "invalid_request_error"),
(418, "api_error"),
]:
with self.subTest(status_code=status_code):
response = self._non_streaming_exception_response(
HTTPException(status_code=status_code, detail="Request rejected")
)
self.assertEqual(response.status_code, status_code)
self.assertEqual(
json.loads(response.body)["error"],
{"type": expected_type, "message": "Request rejected"},
)
def test_non_streaming_http_exception_non_str_detail_is_stringified(self):
response = self._non_streaming_exception_response(
HTTPException(status_code=400, detail={"message": "Bad request"})
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
json.loads(response.body)["error"],
{
"type": "invalid_request_error",
"message": "{'message': 'Bad request'}",
},
)
def test_non_streaming_http_exception_logs_original_detail(self):
detail = "worker rejected request: sensitive internals /opt/secret"
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level="WARNING"
) as captured:
response = self._non_streaming_exception_response(
HTTPException(status_code=503, detail=detail)
)
self.assertEqual(len(captured.records), 1)
message = captured.records[0].getMessage()
self.assertIn(detail, message)
self.assertIn("503", message)
self.assertEqual(response.status_code, 503)
self.assertEqual(
json.loads(response.body)["error"],
{"type": "overloaded_error", "message": "Internal server error"},
)
def test_non_streaming_unexpected_exception_returns_generic_500(self):
response = self._non_streaming_exception_response(
RuntimeError("sensitive internals: /opt/secret")
)
self.assertEqual(response.status_code, 500)
self.assertEqual(
json.loads(response.body)["error"],
{"type": "api_error", "message": "Internal server error"},
)
# ------------------------------------------------------------------
# Edge-case coverage added in the review-fix pass
# ------------------------------------------------------------------