[Responses] Fix empty-prompt routing for token-first chat encoders (kimi_k3, inkling) (#35486)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>

python/sglang/srt/entrypoints/openai/chat_encoding.py

@@ -157,6 +157,15 @@ def spec_owns_reasoning_history(spec: Optional[str]) -> bool:
     return spec is not None
 
 
+def spec_renders_prompt_ids(spec: Optional[str]) -> bool:
+    """Whether the encoder for ``spec`` returns pre-tokenized prompt ids.
+
+    Token-first encoders leave the text prompt empty; the MM processor
+    expands their single placeholder ids rather than re-tokenizing text.
+    """
+    return spec in ("inkling", "kimi_k3")
+
+
 def encode_simple_chat(
     *,
     tokenizer: Any,

python/sglang/srt/entrypoints/openai/serving_chat.py

@@ -1043,6 +1043,22 @@ class OpenAIServingChat(OpenAIServingBase):
             f"received unsupported content type '{media_type}'."
         )
 
+    def _engine_prompt(
+        self, processed_messages: MessageProcessingResult, is_multimodal: bool
+    ) -> tuple[str, Any]:
+        """Standard VLMs render a text prompt (with placeholder strings) for
+        the MM processor to tokenize. Token-first encoders instead produce
+        pre-rendered input_ids with single placeholder ids and leave the text
+        empty; pass those through rather than re-tokenizing an empty prompt.
+        """
+        if is_multimodal and not chat_encoding.spec_renders_prompt_ids(
+            self.chat_encoding_spec
+        ):
+            return "text", processed_messages.prompt
+        if isinstance(processed_messages.prompt_ids, str):
+            return "text", processed_messages.prompt_ids
+        return "input_ids", processed_messages.prompt_ids
+
     def _convert_to_internal_request(
         self,
         request: ChatCompletionRequest,
@@ -1112,27 +1128,11 @@ class OpenAIServingChat(OpenAIServingBase):
         # Handle single vs multiple requests
         if request.input_ids is not None:
             prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
-        elif is_multimodal and self.chat_encoding_spec == "kimi_k3":
-            prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
-        elif is_multimodal:
-            # Standard VLMs render a text prompt (with placeholder strings) for the MM
-            # processor to tokenize. Inkling's custom encoder instead produces pre-rendered
-            # input_ids with single placeholders; pass those through so the MM processor
-            # expands them rather than re-tokenizing an empty prompt. Gated on the Inkling
-            # encoding spec so every other model keeps the standard text path.
-            if (
-                self.chat_encoding_spec == "inkling"
-                and isinstance(processed_messages.prompt_ids, list)
-                and processed_messages.prompt_ids
-            ):
-                prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
         else:
-                prompt_kwargs = {"text": processed_messages.prompt}
+            prompt_key, prompt_value = self._engine_prompt(
-        else:
+                processed_messages, is_multimodal
-            if isinstance(processed_messages.prompt_ids, str):
+            )
-                prompt_kwargs = {"text": processed_messages.prompt_ids}
+            prompt_kwargs = {prompt_key: prompt_value}
-            else:
-                prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
 
         # Extract custom labels from raw request headers
         custom_labels = self.extract_custom_labels(raw_request)

python/sglang/srt/entrypoints/openai/serving_responses.py

@@ -610,12 +610,9 @@ class OpenAIServingResponses(OpenAIServingChat):
             else None
         )
 
-        if is_multimodal:
+        _, engine_prompt = self._engine_prompt(processed_messages, is_multimodal)
-            request_prompts = [processed_messages.prompt]
+        request_prompts = [engine_prompt]
-            engine_prompts = [processed_messages.prompt]
+        engine_prompts = [engine_prompt]
-        else:
-            request_prompts = [processed_messages.prompt_ids]
-            engine_prompts = [processed_messages.prompt_ids]
 
         return messages, request_prompts, engine_prompts, processed_messages
 

test/registered/unit/entrypoints/openai/test_serving_responses.py

@@ -622,6 +622,36 @@ class MultimodalRequestTestCase(CustomTestCase):
         )
         self.assertEqual(captured["adapted_request"].modalities, ["image"])
 
+    def test_multimodal_token_first_specs_route_through_prompt_ids(self):
+        """Bug regression: token-first encoders leave prompt == "" with
+        non-empty prompt_ids; forwarding the empty text 400s in
+        _tokenize_texts, so the multimodal branch must forward prompt_ids."""
+        for spec in ("inkling", "kimi_k3"):
+            with self.subTest(spec=spec):
+                serving = make_serving(is_multimodal=True)
+                serving.chat_encoding_spec = spec
+                serving._process_messages = Mock(
+                    return_value=MessageProcessingResult(
+                        prompt="",
+                        prompt_ids=[4, 5, 6],
+                        image_data=None,
+                        audio_data=None,
+                        video_data=None,
+                        modalities=[],
+                        stop=[],
+                    )
+                )
+                request = ResponsesRequest(model="x", input="hi", store=False)
+
+                _, request_prompts, engine_prompts, _ = asyncio.run(
+                    serving._make_request(
+                        request, None, serving.tokenizer_manager.tokenizer
+                    )
+                )
+
+                self.assertEqual(engine_prompts, [[4, 5, 6]])
+                self.assertEqual(request_prompts, [[4, 5, 6]])
+
 
 class OutputItemsTestCase(CustomTestCase):
     def setUp(self):