diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index f96595fb45..722389b74a 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -12,7 +12,7 @@ import logging import uuid from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Union -from fastapi import Request +from fastapi import HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse from pydantic import BaseModel, ValidationError @@ -757,6 +757,15 @@ class AnthropicServing: ) except asyncio.CancelledError: raise + except HTTPException as e: + logger.warning( + "Non-streaming request rejected: %s %s", e.status_code, e.detail + ) + return self._error_response( + status_code=e.status_code, + error_type=ERROR_TYPE_MAP.get(e.status_code, "api_error"), + message=_scrub_error_message(str(e.detail), e.status_code), + ) except Exception as e: logger.exception("Error processing Anthropic request: %s", e) return self._error_response( diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index 3e78bc76f7..5f8295d6a3 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -1,12 +1,14 @@ import asyncio import json import unittest +from http import HTTPStatus from types import SimpleNamespace from sglang.test.test_utils import maybe_stub_sgl_kernel maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel +from fastapi import HTTPException # noqa: E402 from fastapi.responses import JSONResponse # noqa: E402 from jinja2 import Environment # noqa: E402 @@ -56,14 +58,15 @@ class _FakeOpenAIServingChat: class _FakeNonStreamingErrorOpenAI: - """Returns a configurable error response from the OpenAI handler.""" + """Returns an error response or raises an exception from the OpenAI handler.""" supports_inline_system = False - def __init__(self, status_code=400, body=None, content=None): + def __init__(self, status_code=400, body=None, content=None, exception=None): self._status_code = status_code self._body = body self._content = content + self._exception = exception def _validate_request(self, chat_request): return None @@ -74,6 +77,8 @@ class _FakeNonStreamingErrorOpenAI: async def _handle_non_streaming_request( self, adapted_request, processed_request, raw_request ): + if self._exception is not None: + raise self._exception if self._body is not None: # Build a response object exposing raw bytes via `.body`. return SimpleNamespace(status_code=self._status_code, body=self._body) @@ -89,24 +94,6 @@ class _FakeNonStreamingErrorOpenAI: ) -class _FakeNonStreamingOpenAI: - """Returns a configurable ChatCompletionResponse from the OpenAI handler.""" - - def __init__(self, response): - self._response = response - - def _validate_request(self, chat_request): - return None - - def _convert_to_internal_request(self, chat_request, raw_request): - return SimpleNamespace(), chat_request - - async def _handle_non_streaming_request( - self, adapted_request, processed_request, raw_request - ): - return self._response - - def _chunk(choices=None, usage=None): data = { "id": "chatcmpl-test", @@ -179,6 +166,12 @@ class TestAnthropicServing(unittest.TestCase): data.update(overrides) return AnthropicMessagesRequest.model_validate(data) + def _non_streaming_exception_response(self, exception): + serving = AnthropicServing(_FakeNonStreamingErrorOpenAI(exception=exception)) + return asyncio.run( + serving.handle_messages(self._anthropic_request(stream=False), object()) + ) + def _tool_result_request(self, content, tools=None): overrides = { "stream": False, @@ -551,6 +544,83 @@ class TestAnthropicServing(unittest.TestCase): self.assertEqual(payload["error"]["type"], "invalid_request_error") self.assertEqual(payload["error"]["message"], "context length exceeded") + def test_non_streaming_queue_full_http_exception_preserves_503(self): + for status_code in (503, HTTPStatus.SERVICE_UNAVAILABLE): + with self.subTest(status_code=status_code, code_type=type(status_code)): + response = self._non_streaming_exception_response( + HTTPException( + status_code=status_code, + detail="The request queue is full: sensitive internals /opt/secret", + ) + ) + self.assertEqual(response.status_code, 503) + self.assertEqual( + json.loads(response.body), + { + "type": "error", + "error": { + "type": "overloaded_error", + "message": "Internal server error", + }, + }, + ) + + def test_non_streaming_http_exception_4xx_preserves_status_and_message(self): + for status_code, expected_type in [ + (400, "invalid_request_error"), + (418, "api_error"), + ]: + with self.subTest(status_code=status_code): + response = self._non_streaming_exception_response( + HTTPException(status_code=status_code, detail="Request rejected") + ) + self.assertEqual(response.status_code, status_code) + self.assertEqual( + json.loads(response.body)["error"], + {"type": expected_type, "message": "Request rejected"}, + ) + + def test_non_streaming_http_exception_non_str_detail_is_stringified(self): + response = self._non_streaming_exception_response( + HTTPException(status_code=400, detail={"message": "Bad request"}) + ) + self.assertEqual(response.status_code, 400) + self.assertEqual( + json.loads(response.body)["error"], + { + "type": "invalid_request_error", + "message": "{'message': 'Bad request'}", + }, + ) + + def test_non_streaming_http_exception_logs_original_detail(self): + detail = "worker rejected request: sensitive internals /opt/secret" + with self.assertLogs( + "sglang.srt.entrypoints.anthropic.serving", level="WARNING" + ) as captured: + response = self._non_streaming_exception_response( + HTTPException(status_code=503, detail=detail) + ) + self.assertEqual(len(captured.records), 1) + message = captured.records[0].getMessage() + self.assertIn(detail, message) + self.assertIn("503", message) + self.assertEqual(response.status_code, 503) + self.assertEqual( + json.loads(response.body)["error"], + {"type": "overloaded_error", "message": "Internal server error"}, + ) + + def test_non_streaming_unexpected_exception_returns_generic_500(self): + response = self._non_streaming_exception_response( + RuntimeError("sensitive internals: /opt/secret") + ) + self.assertEqual(response.status_code, 500) + self.assertEqual( + json.loads(response.body)["error"], + {"type": "api_error", "message": "Internal server error"}, + ) + # ------------------------------------------------------------------ # Edge-case coverage added in the review-fix pass # ------------------------------------------------------------------