diff --git a/python/sglang/srt/utils/hf_transformers/mistral_utils.py b/python/sglang/srt/utils/hf_transformers/mistral_utils.py index 8cc17278b..71cd8cc2a 100644 --- a/python/sglang/srt/utils/hf_transformers/mistral_utils.py +++ b/python/sglang/srt/utils/hf_transformers/mistral_utils.py @@ -628,9 +628,48 @@ def patch_mistral_common_tokenizer(tokenizer): adapted.append(msg) return adapted + def _assistant_content_is_empty(content): + if content is None: + return True + if isinstance(content, str): + return not content.strip() + if isinstance(content, list): + return all( + isinstance(part, dict) + and part.get("type") in ("text", "input_text") + and not str(part.get("text") or "").strip() + for part in content + ) + return False + + def _drop_empty_assistant_messages(messages): + """Drop assistant turns with neither content nor tool calls, which + mistral_common rejects while other chat templates ignore them. A trailing + assistant turn is consumed upstream as the continue_final_message prefix, + so this cannot drop a prefill. + """ + if not isinstance(messages, (list, tuple)): + return messages + + kept = [] + for msg in messages: + if isinstance(msg, (list, tuple)): + kept.append(_drop_empty_assistant_messages(msg)) + continue + if ( + isinstance(msg, dict) + and msg.get("role") == "assistant" + and not msg.get("tool_calls") + and _assistant_content_is_empty(msg.get("content")) + ): + continue + kept.append(msg) + return kept + def _safe_apply_chat_template(messages, **kwargs): kwargs.pop("add_generation_prompt", None) messages = _adapt_placeholder_messages_for_mistral_common(messages) + messages = _drop_empty_assistant_messages(messages) return tokenizer._orig_apply_chat_template(messages, **kwargs) tokenizer.apply_chat_template = _safe_apply_chat_template diff --git a/test/registered/unit/tokenizer/test_mistral_empty_assistant.py b/test/registered/unit/tokenizer/test_mistral_empty_assistant.py new file mode 100644 index 000000000..fdcd7a8b7 --- /dev/null +++ b/test/registered/unit/tokenizer/test_mistral_empty_assistant.py @@ -0,0 +1,86 @@ +import unittest + +from sglang.srt.utils.hf_transformers.mistral_utils import ( + patch_mistral_common_tokenizer, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +def _user(text): + return {"role": "user", "content": text} + + +def _assistant(**fields): + return {"role": "assistant", **fields} + + +class _MistralCommonStub: + """Stands in for MistralCommonBackend; the class name gates the patch.""" + + def __init__(self): + self.seen = None + self.chat_template = "x" + + def apply_chat_template(self, messages, **kwargs): + self.seen = messages + return [] + + def add_special_tokens(self, *args, **kwargs): + return 0 + + def convert_tokens_to_ids(self, value): + return 0 + + def decode(self, *args, **kwargs): + return "" + + def batch_decode(self, *args, **kwargs): + return [] + + +class TestDropEmptyAssistantMessages(unittest.TestCase): + def _roles_passed_through(self, messages): + tokenizer = patch_mistral_common_tokenizer(_MistralCommonStub()) + tokenizer.apply_chat_template(messages) + return [msg["role"] for msg in tokenizer.seen] + + def test_empty_assistant_turn_is_dropped(self): + for content in ("", " ", None, [], [{"type": "text", "text": ""}]): + with self.subTest(content=content): + roles = self._roles_passed_through( + [_user("a"), _assistant(content=content), _user("b")] + ) + + self.assertEqual(roles, ["user", "user"]) + + def test_nonempty_assistant_turn_is_kept(self): + for content in ( + "hi", + [{"type": "text", "text": "hi"}], + [ + {"type": "text", "text": ""}, + {"type": "image_url", "image_url": {"url": "https://x/y.png"}}, + ], + ): + with self.subTest(content=content): + roles = self._roles_passed_through( + [_user("a"), _assistant(content=content), _user("b")] + ) + + self.assertEqual(roles, ["user", "assistant", "user"]) + + roles = self._roles_passed_through( + [ + _user("a"), + _assistant(content="", tool_calls=[{"id": "call_1"}]), + _user("b"), + ] + ) + + self.assertEqual(roles, ["user", "assistant", "user"]) + + +if __name__ == "__main__": + unittest.main()