diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding.py b/python/sglang/srt/entrypoints/openai/chat_encoding.py index f70c3e13b..68749449d 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding.py @@ -157,6 +157,15 @@ def spec_owns_reasoning_history(spec: Optional[str]) -> bool: return spec is not None +def spec_renders_prompt_ids(spec: Optional[str]) -> bool: + """Whether the encoder for ``spec`` returns pre-tokenized prompt ids. + + Token-first encoders leave the text prompt empty; the MM processor + expands their single placeholder ids rather than re-tokenizing text. + """ + return spec in ("inkling", "kimi_k3") + + def encode_simple_chat( *, tokenizer: Any, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 75d5f631f..02d19391c 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1043,6 +1043,22 @@ class OpenAIServingChat(OpenAIServingBase): f"received unsupported content type '{media_type}'." ) + def _engine_prompt( + self, processed_messages: MessageProcessingResult, is_multimodal: bool + ) -> tuple[str, Any]: + """Standard VLMs render a text prompt (with placeholder strings) for + the MM processor to tokenize. Token-first encoders instead produce + pre-rendered input_ids with single placeholder ids and leave the text + empty; pass those through rather than re-tokenizing an empty prompt. + """ + if is_multimodal and not chat_encoding.spec_renders_prompt_ids( + self.chat_encoding_spec + ): + return "text", processed_messages.prompt + if isinstance(processed_messages.prompt_ids, str): + return "text", processed_messages.prompt_ids + return "input_ids", processed_messages.prompt_ids + def _convert_to_internal_request( self, request: ChatCompletionRequest, @@ -1112,27 +1128,11 @@ class OpenAIServingChat(OpenAIServingBase): # Handle single vs multiple requests if request.input_ids is not None: prompt_kwargs = {"input_ids": processed_messages.prompt_ids} - elif is_multimodal and self.chat_encoding_spec == "kimi_k3": - prompt_kwargs = {"input_ids": processed_messages.prompt_ids} - elif is_multimodal: - # Standard VLMs render a text prompt (with placeholder strings) for the MM - # processor to tokenize. Inkling's custom encoder instead produces pre-rendered - # input_ids with single placeholders; pass those through so the MM processor - # expands them rather than re-tokenizing an empty prompt. Gated on the Inkling - # encoding spec so every other model keeps the standard text path. - if ( - self.chat_encoding_spec == "inkling" - and isinstance(processed_messages.prompt_ids, list) - and processed_messages.prompt_ids - ): - prompt_kwargs = {"input_ids": processed_messages.prompt_ids} - else: - prompt_kwargs = {"text": processed_messages.prompt} else: - if isinstance(processed_messages.prompt_ids, str): - prompt_kwargs = {"text": processed_messages.prompt_ids} - else: - prompt_kwargs = {"input_ids": processed_messages.prompt_ids} + prompt_key, prompt_value = self._engine_prompt( + processed_messages, is_multimodal + ) + prompt_kwargs = {prompt_key: prompt_value} # Extract custom labels from raw request headers custom_labels = self.extract_custom_labels(raw_request) diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 799211d9f..cdbc2eb38 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -610,12 +610,9 @@ class OpenAIServingResponses(OpenAIServingChat): else None ) - if is_multimodal: - request_prompts = [processed_messages.prompt] - engine_prompts = [processed_messages.prompt] - else: - request_prompts = [processed_messages.prompt_ids] - engine_prompts = [processed_messages.prompt_ids] + _, engine_prompt = self._engine_prompt(processed_messages, is_multimodal) + request_prompts = [engine_prompt] + engine_prompts = [engine_prompt] return messages, request_prompts, engine_prompts, processed_messages diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses.py b/test/registered/unit/entrypoints/openai/test_serving_responses.py index b9d9aca75..bac16c62c 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses.py @@ -622,6 +622,36 @@ class MultimodalRequestTestCase(CustomTestCase): ) self.assertEqual(captured["adapted_request"].modalities, ["image"]) + def test_multimodal_token_first_specs_route_through_prompt_ids(self): + """Bug regression: token-first encoders leave prompt == "" with + non-empty prompt_ids; forwarding the empty text 400s in + _tokenize_texts, so the multimodal branch must forward prompt_ids.""" + for spec in ("inkling", "kimi_k3"): + with self.subTest(spec=spec): + serving = make_serving(is_multimodal=True) + serving.chat_encoding_spec = spec + serving._process_messages = Mock( + return_value=MessageProcessingResult( + prompt="", + prompt_ids=[4, 5, 6], + image_data=None, + audio_data=None, + video_data=None, + modalities=[], + stop=[], + ) + ) + request = ResponsesRequest(model="x", input="hi", store=False) + + _, request_prompts, engine_prompts, _ = asyncio.run( + serving._make_request( + request, None, serving.tokenizer_manager.tokenizer + ) + ) + + self.assertEqual(engine_prompts, [[4, 5, 6]]) + self.assertEqual(request_prompts, [[4, 5, 6]]) + class OutputItemsTestCase(CustomTestCase): def setUp(self):