diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 57a02681c..eef87d25c 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -204,6 +204,12 @@ class BaseMultimodalProcessor(ABC): else: self._tokenizer = self._processor + # Same guard as in serving_chat.py against double BOS. + try: + self._tokenizer_auto_adds_specials = len(self._tokenizer.encode("")) > 0 + except Exception: + self._tokenizer_auto_adds_specials = False + # FIXME: not accurate, model and image specific self.NUM_TOKEN_PER_FRAME = 330 @@ -467,6 +473,12 @@ class BaseMultimodalProcessor(ABC): npu_apply_glm46v_image_preprocess_patch() kwargs["device"] = "npu" + # Avoid double BOS when the chat template already wrote one. + if self._tokenizer_auto_adds_specials and isinstance(input_text, str): + bos = getattr(self._tokenizer, "bos_token", None) + if bos and input_text.startswith(bos): + kwargs.setdefault("add_special_tokens", False) + result = processor.__call__( text=[input_text], padding=True, diff --git a/test/registered/unit/managers/test_mm_process_config.py b/test/registered/unit/managers/test_mm_process_config.py index 253cde944..71628f55a 100644 --- a/test/registered/unit/managers/test_mm_process_config.py +++ b/test/registered/unit/managers/test_mm_process_config.py @@ -138,6 +138,8 @@ class TestProcessMmDataKwargs(unittest.TestCase): proc.disable_fast_image_processor = server_args.disable_fast_image_processor proc.skip_tokenizer_init = server_args.skip_tokenizer_init proc._processor = mock_processor + proc._tokenizer = MagicMock() + proc._tokenizer_auto_adds_specials = False proc.image_config = mm_process_config.get("image", {}) proc.video_config = mm_process_config.get("video", {}) proc.audio_config = mm_process_config.get("audio", {}) @@ -296,5 +298,48 @@ class TestOverrideProcessorsConfigInjection(unittest.TestCase): self.assertTrue(audio_kw.get("truncation")) +class TestDoubleBosGuard(unittest.TestCase): + """Regression test for the multimodal double-BOS bug. + + Repro condition (Cohere2 / Llama3-LLaVA-Next family): + - tokenizer.encode("") returns [bos_id] (auto-adds specials), AND + - chat template renders the BOS string as a literal at the start. + + Without the guard in BaseMultimodalProcessor, the inner processor.__call__ + on the rendered prompt would auto-prepend a second BOS, producing 2 leading + BOS tokens vs the HF reference's 1. + """ + + def test_guard_passes_add_special_tokens_false_on_bug_condition(self): + from sglang.srt.multimodal.processors.base_processor import ( + BaseMultimodalProcessor, + ) + + server_args = MagicMock() + server_args.mm_process_config = {} + server_args.disable_fast_image_processor = True + server_args.keep_mm_feature_on_device = True + + mock_hf_processor = MagicMock() + mock_hf_processor.__class__.__name__ = "TestProcessor" + mock_hf_processor.__call__ = MagicMock(return_value={}) + mock_hf_processor.tokenizer.encode = MagicMock(return_value=[2]) + mock_hf_processor.tokenizer.bos_token = "" + + with patch.object(BaseMultimodalProcessor, "__abstractmethods__", set()): + proc = BaseMultimodalProcessor( + hf_config=MagicMock(), + server_args=server_args, + _processor=mock_hf_processor, + transport_mode=None, + ) + proc.FEATURE_NAMES = [] + + proc.process_mm_data("hello", images=["img1"]) + + call_kwargs = mock_hf_processor.__call__.call_args.kwargs + self.assertEqual(call_kwargs.get("add_special_tokens"), False) + + if __name__ == "__main__": unittest.main()