[Fix] Preserve HTTP errors for non-streaming /v1/messages (#43796)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
co-authored by
Xinyuan Tong
Xinyuan Tong
parent
45418d7dc6
commit
75a0d91c1f
@@ -12,7 +12,7 @@ import logging
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Union
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
@@ -757,6 +757,15 @@ class AnthropicServing:
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except HTTPException as e:
|
||||
logger.warning(
|
||||
"Non-streaming request rejected: %s %s", e.status_code, e.detail
|
||||
)
|
||||
return self._error_response(
|
||||
status_code=e.status_code,
|
||||
error_type=ERROR_TYPE_MAP.get(e.status_code, "api_error"),
|
||||
message=_scrub_error_message(str(e.detail), e.status_code),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error processing Anthropic request: %s", e)
|
||||
return self._error_response(
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import asyncio
|
||||
import json
|
||||
import unittest
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel
|
||||
|
||||
from fastapi import HTTPException # noqa: E402
|
||||
from fastapi.responses import JSONResponse # noqa: E402
|
||||
from jinja2 import Environment # noqa: E402
|
||||
|
||||
@@ -56,14 +58,15 @@ class _FakeOpenAIServingChat:
|
||||
|
||||
|
||||
class _FakeNonStreamingErrorOpenAI:
|
||||
"""Returns a configurable error response from the OpenAI handler."""
|
||||
"""Returns an error response or raises an exception from the OpenAI handler."""
|
||||
|
||||
supports_inline_system = False
|
||||
|
||||
def __init__(self, status_code=400, body=None, content=None):
|
||||
def __init__(self, status_code=400, body=None, content=None, exception=None):
|
||||
self._status_code = status_code
|
||||
self._body = body
|
||||
self._content = content
|
||||
self._exception = exception
|
||||
|
||||
def _validate_request(self, chat_request):
|
||||
return None
|
||||
@@ -74,6 +77,8 @@ class _FakeNonStreamingErrorOpenAI:
|
||||
async def _handle_non_streaming_request(
|
||||
self, adapted_request, processed_request, raw_request
|
||||
):
|
||||
if self._exception is not None:
|
||||
raise self._exception
|
||||
if self._body is not None:
|
||||
# Build a response object exposing raw bytes via `.body`.
|
||||
return SimpleNamespace(status_code=self._status_code, body=self._body)
|
||||
@@ -89,24 +94,6 @@ class _FakeNonStreamingErrorOpenAI:
|
||||
)
|
||||
|
||||
|
||||
class _FakeNonStreamingOpenAI:
|
||||
"""Returns a configurable ChatCompletionResponse from the OpenAI handler."""
|
||||
|
||||
def __init__(self, response):
|
||||
self._response = response
|
||||
|
||||
def _validate_request(self, chat_request):
|
||||
return None
|
||||
|
||||
def _convert_to_internal_request(self, chat_request, raw_request):
|
||||
return SimpleNamespace(), chat_request
|
||||
|
||||
async def _handle_non_streaming_request(
|
||||
self, adapted_request, processed_request, raw_request
|
||||
):
|
||||
return self._response
|
||||
|
||||
|
||||
def _chunk(choices=None, usage=None):
|
||||
data = {
|
||||
"id": "chatcmpl-test",
|
||||
@@ -179,6 +166,12 @@ class TestAnthropicServing(unittest.TestCase):
|
||||
data.update(overrides)
|
||||
return AnthropicMessagesRequest.model_validate(data)
|
||||
|
||||
def _non_streaming_exception_response(self, exception):
|
||||
serving = AnthropicServing(_FakeNonStreamingErrorOpenAI(exception=exception))
|
||||
return asyncio.run(
|
||||
serving.handle_messages(self._anthropic_request(stream=False), object())
|
||||
)
|
||||
|
||||
def _tool_result_request(self, content, tools=None):
|
||||
overrides = {
|
||||
"stream": False,
|
||||
@@ -551,6 +544,83 @@ class TestAnthropicServing(unittest.TestCase):
|
||||
self.assertEqual(payload["error"]["type"], "invalid_request_error")
|
||||
self.assertEqual(payload["error"]["message"], "context length exceeded")
|
||||
|
||||
def test_non_streaming_queue_full_http_exception_preserves_503(self):
|
||||
for status_code in (503, HTTPStatus.SERVICE_UNAVAILABLE):
|
||||
with self.subTest(status_code=status_code, code_type=type(status_code)):
|
||||
response = self._non_streaming_exception_response(
|
||||
HTTPException(
|
||||
status_code=status_code,
|
||||
detail="The request queue is full: sensitive internals /opt/secret",
|
||||
)
|
||||
)
|
||||
self.assertEqual(response.status_code, 503)
|
||||
self.assertEqual(
|
||||
json.loads(response.body),
|
||||
{
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "overloaded_error",
|
||||
"message": "Internal server error",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def test_non_streaming_http_exception_4xx_preserves_status_and_message(self):
|
||||
for status_code, expected_type in [
|
||||
(400, "invalid_request_error"),
|
||||
(418, "api_error"),
|
||||
]:
|
||||
with self.subTest(status_code=status_code):
|
||||
response = self._non_streaming_exception_response(
|
||||
HTTPException(status_code=status_code, detail="Request rejected")
|
||||
)
|
||||
self.assertEqual(response.status_code, status_code)
|
||||
self.assertEqual(
|
||||
json.loads(response.body)["error"],
|
||||
{"type": expected_type, "message": "Request rejected"},
|
||||
)
|
||||
|
||||
def test_non_streaming_http_exception_non_str_detail_is_stringified(self):
|
||||
response = self._non_streaming_exception_response(
|
||||
HTTPException(status_code=400, detail={"message": "Bad request"})
|
||||
)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(
|
||||
json.loads(response.body)["error"],
|
||||
{
|
||||
"type": "invalid_request_error",
|
||||
"message": "{'message': 'Bad request'}",
|
||||
},
|
||||
)
|
||||
|
||||
def test_non_streaming_http_exception_logs_original_detail(self):
|
||||
detail = "worker rejected request: sensitive internals /opt/secret"
|
||||
with self.assertLogs(
|
||||
"sglang.srt.entrypoints.anthropic.serving", level="WARNING"
|
||||
) as captured:
|
||||
response = self._non_streaming_exception_response(
|
||||
HTTPException(status_code=503, detail=detail)
|
||||
)
|
||||
self.assertEqual(len(captured.records), 1)
|
||||
message = captured.records[0].getMessage()
|
||||
self.assertIn(detail, message)
|
||||
self.assertIn("503", message)
|
||||
self.assertEqual(response.status_code, 503)
|
||||
self.assertEqual(
|
||||
json.loads(response.body)["error"],
|
||||
{"type": "overloaded_error", "message": "Internal server error"},
|
||||
)
|
||||
|
||||
def test_non_streaming_unexpected_exception_returns_generic_500(self):
|
||||
response = self._non_streaming_exception_response(
|
||||
RuntimeError("sensitive internals: /opt/secret")
|
||||
)
|
||||
self.assertEqual(response.status_code, 500)
|
||||
self.assertEqual(
|
||||
json.loads(response.body)["error"],
|
||||
{"type": "api_error", "message": "Internal server error"},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Edge-case coverage added in the review-fix pass
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user