diff --git a/python/sglang/benchmark/datasets/image.py b/python/sglang/benchmark/datasets/image.py index e55194f2e..5deffe0a4 100644 --- a/python/sglang/benchmark/datasets/image.py +++ b/python/sglang/benchmark/datasets/image.py @@ -199,22 +199,23 @@ def create_mm_data_row( # Vision tokens = total tokens - text tokens vision_prompt_len = prompt_len - text_prompt_len - supported_backends = [ + supported_backends = ( "sglang", "sglang-native", "sglang-oai-chat", "vllm-chat", - ] + "lmdeploy-chat", + ) if backend not in supported_backends: raise ValueError( f"Image dataset only supports backends: {supported_backends}, " f"got '{backend}'." ) - # OpenAI chat handlers apply the chat template and receive images separately, so - # send the raw text. /generate does not apply a chat template, so it needs - # prompt_str, which contains the multimodal processor's image placeholders. - use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat") + # Chat-completions backends apply their own chat template, so send raw text. + # Native SGLang /generate does not apply a template and needs the image + # placeholder-bearing prompt generated by the processor. + use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat", "lmdeploy-chat") return DatasetRow( prompt=text_prompt if use_raw_prompt else prompt_str, diff --git a/test/registered/bench_fn/test_benchmark_datasets_api.py b/test/registered/bench_fn/test_benchmark_datasets_api.py index 2f7ca24d5..4c7c3d7f4 100644 --- a/test/registered/bench_fn/test_benchmark_datasets_api.py +++ b/test/registered/bench_fn/test_benchmark_datasets_api.py @@ -426,24 +426,26 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase): self.assertTrue(all(isinstance(row, DatasetRow) for row in rows)) self.assertTrue(all(row.image_data for row in rows)) - def test_image_sampler_vllm_chat(self): - rows = sample_image_requests( - num_requests=2, - image_count=1, - input_len=8, - output_len=4, - range_ratio=0.0, - processor=self.processor, - image_content="blank", - image_format="png", - image_resolution="8x8", - backend="vllm-chat", - random_image_count=False, - ) - self.assertEqual(len(rows), 2) - self.assertTrue(all(isinstance(row, DatasetRow) for row in rows)) - self.assertTrue(all(row.image_data for row in rows)) - self.assertTrue(all("[IMAGE]" not in row.prompt for row in rows)) + def test_image_sampler_chat_backends_use_raw_prompt(self): + for backend in ("sglang-oai-chat", "vllm-chat", "lmdeploy-chat"): + with self.subTest(backend=backend): + rows = sample_image_requests( + num_requests=1, + image_count=1, + input_len=8, + output_len=4, + range_ratio=0.0, + processor=self.processor, + image_content="blank", + image_format="png", + image_resolution="8x8", + backend=backend, + random_image_count=False, + ) + self.assertEqual(len(rows), 1) + self.assertTrue(rows[0].image_data) + for marker in ("user:", "assistant:", "[IMAGE]"): + self.assertNotIn(marker, rows[0].prompt) def test_image_sampler_random_resolution(self): state = np.random.get_state()