diff --git a/docs/cookbook/autoregressive/Google/Gemma4.mdx b/docs/cookbook/autoregressive/Google/Gemma4.mdx index f383329bf..946fd5dc0 100644 --- a/docs/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs/cookbook/autoregressive/Google/Gemma4.mdx @@ -531,7 +531,7 @@ Tool Call: get_weather ### 4.5 Audio Input -The audio-capable Gemma 4 variants (`gemma-4-E2B-it`, `gemma-4-E4B-it`, `gemma-4-12B-it`) accept raw audio alongside text. Pass the waveform as a base64 `audio_url` data URI (16 kHz mono WAV works well): +The audio-capable Gemma 4 variants (`gemma-4-E2B-it`, `gemma-4-E4B-it`, `gemma-4-12B-it`) accept raw audio alongside text. Pass the waveform as a base64 `audio_url` data URI (16 kHz mono WAV works well), or as OpenAI's `input_audio` part with the base64 bytes in `data` and a `format` of `wav`: ```python Example import base64 diff --git a/docs/cookbook/autoregressive/ThinkingMachines/Inkling-Small.mdx b/docs/cookbook/autoregressive/ThinkingMachines/Inkling-Small.mdx index d95e1ecab..5bc7305aa 100644 --- a/docs/cookbook/autoregressive/ThinkingMachines/Inkling-Small.mdx +++ b/docs/cookbook/autoregressive/ThinkingMachines/Inkling-Small.mdx @@ -216,7 +216,7 @@ Tool calls: [ChatCompletionMessageFunctionToolCall(id='call_98f772f3a0044f45b80c ### 3.3 Multimodal Input (Image + Audio) -Inkling-Small is multimodal: a single user message can mix **text**, **images**, and **audio**. Pass each media item as its own content part — `image_url` for images, `audio_url` for audio — with the `url` set to either an HTTP(S) link or a base64 `data:` URI. The server must be started with `--enable-multimodal` (already included in every recipe above). +Inkling-Small is multimodal: a single user message can mix **text**, **images**, and **audio**. Pass each media item as its own content part — `image_url` for images, `audio_url` for audio — with the `url` set to either an HTTP(S) link or a base64 `data:` URI. Audio can also be sent as OpenAI's `input_audio` part, carrying the base64 bytes in `data` alongside a `format` of `wav` or `mp3`. The server must be started with `--enable-multimodal` (already included in every recipe above). diff --git a/docs/cookbook/autoregressive/ThinkingMachines/Inkling.mdx b/docs/cookbook/autoregressive/ThinkingMachines/Inkling.mdx index 3ef94f69b..b7a74b979 100644 --- a/docs/cookbook/autoregressive/ThinkingMachines/Inkling.mdx +++ b/docs/cookbook/autoregressive/ThinkingMachines/Inkling.mdx @@ -213,7 +213,7 @@ Tool calls: [ChatCompletionMessageFunctionToolCall(id='call_98f772f3a0044f45b80c ### 3.3 Multimodal Input (Image + Audio) -Inkling is multimodal: a single user message can mix **text**, **images**, and **audio**. Pass each media item as its own content part — `image_url` for images, `audio_url` for audio — with the `url` set to either an HTTP(S) link or a base64 `data:` URI. The server must be started with `--enable-multimodal` (already included in every recipe above). +Inkling is multimodal: a single user message can mix **text**, **images**, and **audio**. Pass each media item as its own content part — `image_url` for images, `audio_url` for audio — with the `url` set to either an HTTP(S) link or a base64 `data:` URI. Audio can also be sent as OpenAI's `input_audio` part, carrying the base64 bytes in `data` alongside a `format` of `wav` or `mp3`. The server must be started with `--enable-multimodal` (already included in every recipe above). diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 4c98638eb..656fe73e8 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -50,6 +50,7 @@ from openai.types.responses.response_format_text_json_schema_config import ( ) from openai.types.shared.response_format_json_object import ResponseFormatJSONObject from pydantic import ( + AfterValidator, BaseModel, ConfigDict, Field, @@ -582,11 +583,57 @@ class ChatCompletionMessageContentVideoPart(BaseModel): video_url: ChatCompletionMessageContentVideoURL -class ChatCompletionMessageContentAudioPart(BaseModel): +class ChatCompletionMessageContentInputAudio(BaseModel): + data: str + format: Literal["wav", "mp3"] + + +_AUDIO_FORMAT_TO_MIME_TYPE = { + "wav": "audio/wav", + "mp3": "audio/mpeg", +} + + +class ChatCompletionMessageContentAudioURLPart(BaseModel): type: Literal["audio_url"] audio_url: ChatCompletionMessageContentAudioURL +class ChatCompletionMessageContentAudioInlinePart(BaseModel): + type: Literal["input_audio"] + input_audio: ChatCompletionMessageContentInputAudio + + +def _to_audio_url_part( + part: Union[ + ChatCompletionMessageContentAudioURLPart, + ChatCompletionMessageContentAudioInlinePart, + ], +) -> ChatCompletionMessageContentAudioURLPart: + if isinstance(part, ChatCompletionMessageContentAudioURLPart): + return part + + audio = part.input_audio + return ChatCompletionMessageContentAudioURLPart( + type="audio_url", + audio_url=ChatCompletionMessageContentAudioURL( + url=f"data:{_AUDIO_FORMAT_TO_MIME_TYPE[audio.format]};base64,{audio.data}" + ), + ) + + +# Audio arrives by reference as `audio_url`, holding a URL or a data URI, or +# inline as OpenAI's `input_audio`, holding base64. Inline audio is converted to +# the equivalent data URI as it validates. +ChatCompletionMessageContentAudioPart = Annotated[ + Union[ + ChatCompletionMessageContentAudioURLPart, + ChatCompletionMessageContentAudioInlinePart, + ], + AfterValidator(_to_audio_url_part), +] + + class ChatCompletionMessageContentToolReferenceBlock(BaseModel): # GLM-specific extension used alongside `defer_loading` tools. The chat # template looks up `tools[*].function.name == tr.name` and renders the diff --git a/test/registered/unit/entrypoints/openai/test_protocol.py b/test/registered/unit/entrypoints/openai/test_protocol.py index fd663bdec..dda22cd44 100644 --- a/test/registered/unit/entrypoints/openai/test_protocol.py +++ b/test/registered/unit/entrypoints/openai/test_protocol.py @@ -16,9 +16,11 @@ import unittest from typing import List, Optional -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, Field, TypeAdapter, ValidationError from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionMessageContentAudioPart, + ChatCompletionMessageContentAudioURLPart, ChatCompletionMessageContentImageURL, ChatCompletionRequest, ChatCompletionResponse, @@ -550,6 +552,75 @@ class TestChatCompletionRequest(unittest.TestCase): self.assertNotIn("json_schema", sampling_params) +class TestAudioContentParts(unittest.TestCase): + """Test audio content parts and the input_audio conversion""" + + def _audio_part(self, part): + """Validate a content part the way a request body would deliver it.""" + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": [part]}], + ) + return request.messages[0].content[0] + + def test_input_audio_converted_to_data_uri(self): + part = self._audio_part( + {"type": "input_audio", "input_audio": {"data": "QUJD", "format": "wav"}} + ) + # Converted during validation, so the inline type does not survive. + self.assertIsInstance(part, ChatCompletionMessageContentAudioURLPart) + self.assertEqual(part.type, "audio_url") + self.assertEqual(part.audio_url.url, "data:audio/wav;base64,QUJD") + + def test_input_audio_mp3_uses_registered_mime_type(self): + part = self._audio_part( + {"type": "input_audio", "input_audio": {"data": "QUJD", "format": "mp3"}} + ) + self.assertEqual(part.audio_url.url, "data:audio/mpeg;base64,QUJD") + + def test_audio_url_passes_through_unchanged(self): + for url in ("http://example.com/audio.wav", "data:audio/wav;base64,QUJD"): + with self.subTest(url=url): + part = self._audio_part( + {"type": "audio_url", "audio_url": {"url": url}} + ) + self.assertEqual(part.type, "audio_url") + self.assertEqual(part.audio_url.url, url) + + def test_input_audio_rejects_unsupported_format(self): + with self.assertRaises(ValidationError): + self._audio_part( + { + "type": "input_audio", + "input_audio": {"data": "QUJD", "format": "ogg"}, + } + ) + + def test_input_audio_requires_a_payload(self): + with self.assertRaises(ValidationError): + self._audio_part({"type": "input_audio"}) + + def test_audio_url_requires_a_payload(self): + with self.assertRaises(ValidationError): + self._audio_part({"type": "audio_url"}) + + def test_schema_advertises_both_spellings(self): + """Accepting input_audio without publishing it would hide the feature. + + Each variant requires its own payload, so the schema states that exactly + one of the two forms is expected rather than leaving both optional. + """ + schema = TypeAdapter(ChatCompletionMessageContentAudioPart).json_schema() + variants = { + frozenset(schema["$defs"][ref["$ref"].rsplit("/", 1)[-1]]["required"]) + for ref in schema["anyOf"] + } + self.assertEqual( + variants, + {frozenset({"type", "audio_url"}), frozenset({"type", "input_audio"})}, + ) + + class TestModelSerialization(unittest.TestCase): """Test model serialization with hidden states""" diff --git a/test/registered/unit/parser/test_conversation.py b/test/registered/unit/parser/test_conversation.py index da225fea7..dcc3b0040 100644 --- a/test/registered/unit/parser/test_conversation.py +++ b/test/registered/unit/parser/test_conversation.py @@ -6,8 +6,8 @@ import tempfile import unittest from sglang.srt.entrypoints.openai.protocol import ( - ChatCompletionMessageContentAudioPart, ChatCompletionMessageContentAudioURL, + ChatCompletionMessageContentAudioURLPart, ChatCompletionMessageContentImagePart, ChatCompletionMessageContentImageURL, ChatCompletionMessageContentTextPart, @@ -911,7 +911,7 @@ class TestGenerateChatConv(CustomTestCase): ChatCompletionMessageContentTextPart( type="text", text="Transcribe this" ), - ChatCompletionMessageContentAudioPart( + ChatCompletionMessageContentAudioURLPart( type="audio_url", audio_url=ChatCompletionMessageContentAudioURL( url="http://example.com/audio.wav" @@ -925,6 +925,31 @@ class TestGenerateChatConv(CustomTestCase): self.assertEqual(len(conv.audio_data), 1) self.assertEqual(conv.audio_data[0], "http://example.com/audio.wav") + def test_user_message_with_inline_audio(self): + """Inline input_audio reaches the parser as a data URI. + + Built from raw dicts so the content parts go through validation the way + a request body does, which is where the conversion happens; the parser + itself only knows about `audio_url`. + """ + request = self._make_request( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": "QUJD", "format": "wav"}, + }, + ], + } + ] + ) + conv = generate_chat_conv(request, "chatml") + self.assertEqual(len(conv.audio_data), 1) + self.assertEqual(conv.audio_data[0], "data:audio/wav;base64,QUJD") + def test_user_message_image_at_prefix(self): """Test image_token_at_prefix=True puts image token before text.""" # Register a temporary template with image_token_at_prefix=True