From 5e7190b87ab483b9652c108ea4b65a816ea60f71 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 19 Oct 2026 10:12:16 +0800 Subject: [PATCH] PR #40758: escape literal DSV4.1 image placeholder in user text Source: https://github.com/sgl-project/sglang/pull/40758 (applied via GitHub .diff, 3way) --- .../srt/entrypoints/openai/encoding_dsv41.py | 59 +++++++-- .../entrypoints/openai/test_encoding_dsv41.py | 115 ++++++++++++++++++ 2 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 test/registered/unit/entrypoints/openai/test_encoding_dsv41.py diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv41.py b/python/sglang/srt/entrypoints/openai/encoding_dsv41.py index c04d8059d8..516549fcdc 100644 --- a/python/sglang/srt/entrypoints/openai/encoding_dsv41.py +++ b/python/sglang/srt/entrypoints/openai/encoding_dsv41.py @@ -25,6 +25,12 @@ LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" IMAGE_PLACEHOLDER = "<|deepseek_image|>" +# ASCII-pipe spelling of IMAGE_PLACEHOLDER. User-supplied text that happens to +# contain the literal fullwidth-bar token is rewritten to this spelling +# instead of being rejected, so it can never be confused with a genuine +# image placeholder inserted by _process_image_blocks for real image content. +IMAGE_PLACEHOLDER_ESCAPED = "<|deepseek_image|>" + # Task special tokens for internal classification tasks DS_TASK_SP_TOKENS = { "action": "<|action|>", @@ -528,9 +534,26 @@ def _extract_image(block: Dict[str, Any]) -> Dict[str, Any]: return {"url": url} +def _escape_image_placeholder(text: str) -> str: + """Rewrite a literal image-placeholder token to its ASCII-pipe spelling. + + Applied to user-supplied text (never to a placeholder this module inserts + itself for a genuine image block), so the escaped spelling can never be + mistaken for a real image placeholder downstream. + """ + return text.replace(IMAGE_PLACEHOLDER, IMAGE_PLACEHOLDER_ESCAPED) + + def _process_image_blocks( blocks: List[Any], image_placeholder: str = IMAGE_PLACEHOLDER ) -> Tuple[List[Any], List[Dict[str, Any]]]: + """Replace image blocks with placeholders and collect their records in order. + + Any literal image-placeholder token found in user-supplied text (a text + block, or a tool-result block's text content) is escaped to its + ASCII-pipe spelling rather than rejected; only a genuine image/image_url + block inserts the real placeholder. + """ new_blocks: List[Any] = [] images: List[Dict[str, Any]] = [] for block in blocks: @@ -549,31 +572,41 @@ def _process_image_blocks( ) new_blocks.append(block) images.extend(nested_images) + elif block.get("type") == "tool_result" and isinstance( + block.get("content"), str + ): + content = block.get("content", "") + if IMAGE_PLACEHOLDER in content: + block = copy.copy(block) + block["content"] = _escape_image_placeholder(content) + new_blocks.append(block) elif block.get("type") == "text": text = block.get("text") or "" if IMAGE_PLACEHOLDER in text: - raise ValueError( - f"Text block contains image placeholder '{IMAGE_PLACEHOLDER}': " - f"'{text[:100]}'. Images should be separate content blocks." - ) + block = copy.copy(block) + block["text"] = _escape_image_placeholder(text) new_blocks.append(block) else: new_blocks.append(block) return new_blocks, images -def _validate_no_image_sp_tokens(msg: Dict[str, Any]) -> None: +def _escape_image_sp_tokens(msg: Dict[str, Any]) -> None: + """Escape user-supplied image placeholder tokens in textual fields. + + A literal image-placeholder token typed or echoed back by the user (in + ``content``, ``reasoning_content``, or a tool message's ``content``) is + rewritten in place to its ASCII-pipe spelling instead of being rejected, + so it can never be mistaken for a genuine image placeholder this module + inserts for real image content. + """ content = msg.get("content") if isinstance(content, str) and IMAGE_PLACEHOLDER in content: - raise ValueError( - f"Message content contains image special token '{IMAGE_PLACEHOLDER}'. " - "Images should be provided as image content blocks." - ) + msg["content"] = _escape_image_placeholder(content) + reasoning_content = msg.get("reasoning_content") if isinstance(reasoning_content, str) and IMAGE_PLACEHOLDER in reasoning_content: - raise ValueError( - f"reasoning_content contains image special token '{IMAGE_PLACEHOLDER}'" - ) + msg["reasoning_content"] = _escape_image_placeholder(reasoning_content) def process_image_messages( @@ -583,7 +616,7 @@ def process_image_messages( images: List[Dict[str, Any]] = [] for msg in messages: msg = copy.deepcopy(msg) - _validate_no_image_sp_tokens(msg) + _escape_image_sp_tokens(msg) if isinstance(msg.get("content"), list) and "content_blocks" not in msg: msg["content_blocks"] = msg.pop("content") diff --git a/test/registered/unit/entrypoints/openai/test_encoding_dsv41.py b/test/registered/unit/entrypoints/openai/test_encoding_dsv41.py new file mode 100644 index 0000000000..d6601be179 --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_encoding_dsv41.py @@ -0,0 +1,115 @@ +"""Unit tests for the DeepSeek-V4.1 chat encoder -- no server, no model loading.""" + +import unittest + +from sglang.srt.entrypoints.openai.encoding_dsv41 import ( + IMAGE_PLACEHOLDER, + IMAGE_PLACEHOLDER_ESCAPED, + encode_messages, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestImagePlaceholderEscaping(CustomTestCase): + """A literal '<|deepseek_image|>' typed by the user must never be mistaken + for a real image placeholder: it is escaped to the ASCII-pipe spelling + instead of being rejected, and it must never produce an image token. + """ + + def test_tool_result_literal_placeholder_is_escaped_not_rejected(self): + messages = [ + {"role": "user", "content": "question"}, + { + "role": "assistant", + "content": "calling tool", + "tool_calls": [ + { + "id": "a", + "type": "function", + "function": {"name": "lookup", "arguments": {"query": "q"}}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "a", + "content": f"result mentions {IMAGE_PLACEHOLDER} literally", + }, + ] + prompt, mm = encode_messages( + messages, thinking_mode="chat", return_multi_modal_data=True + ) + self.assertEqual(mm["images"], []) + self.assertNotIn(IMAGE_PLACEHOLDER, prompt) + self.assertIn(IMAGE_PLACEHOLDER_ESCAPED, prompt) + + def test_user_text_literal_placeholder_is_escaped_not_rejected(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"please explain {IMAGE_PLACEHOLDER}"} + ], + }, + ] + prompt, mm = encode_messages( + messages, thinking_mode="chat", return_multi_modal_data=True + ) + self.assertEqual(mm["images"], []) + self.assertNotIn(IMAGE_PLACEHOLDER, prompt) + self.assertIn(IMAGE_PLACEHOLDER_ESCAPED, prompt) + + def test_string_content_literal_placeholder_is_escaped_not_rejected(self): + messages = [ + { + "role": "user", + "content": f"What does the token {IMAGE_PLACEHOLDER} mean?", + }, + ] + prompt, mm = encode_messages( + messages, thinking_mode="thinking", return_multi_modal_data=True + ) + self.assertEqual(mm["images"], []) + self.assertNotIn(IMAGE_PLACEHOLDER, prompt) + self.assertIn(IMAGE_PLACEHOLDER_ESCAPED, prompt) + + def test_reasoning_content_literal_placeholder_is_escaped(self): + messages = [ + {"role": "user", "content": "question"}, + { + "role": "assistant", + "reasoning_content": f"thinking about {IMAGE_PLACEHOLDER}", + "content": "answer", + }, + ] + prompt = encode_messages( + messages, thinking_mode="thinking", drop_thinking=False + ) + self.assertNotIn(IMAGE_PLACEHOLDER, prompt) + self.assertIn(IMAGE_PLACEHOLDER_ESCAPED, prompt) + + def test_real_image_block_still_inserts_the_actual_placeholder(self): + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.png"}, + }, + {"type": "text", "text": "what is this?"}, + ], + }, + ] + prompt, mm = encode_messages( + messages, thinking_mode="chat", return_multi_modal_data=True + ) + self.assertEqual(len(mm["images"]), 1) + self.assertIn(IMAGE_PLACEHOLDER, prompt) + + +if __name__ == "__main__": + unittest.main()