[Fix] Resolve VLM test image placeholders from the model's own chat template (#33509)

python/sglang/test/test_utils.py

@@ -183,6 +183,22 @@ def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Ima
             time.sleep(2**i)
 
 
+def build_vlm_image_prompt(processor, question: str) -> str:
+    # Take the image placeholder from the model's own HF chat template: a
+    # hand-written one silently degrades to a text-only prompt on any model
+    # whose placeholder differs.
+    return processor.apply_chat_template(
+        [
+            {
+                "role": "user",
+                "content": [{"type": "image"}, {"type": "text", "text": question}],
+            }
+        ],
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+
+
 def is_in_ci():
     """Return whether it is in CI runner."""
     return get_bool_env_var("SGLANG_IS_IN_CI")

test/manual/distributed/test_dp_attention_large.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import requests
 
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
 from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
 from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
@@ -164,24 +163,38 @@ class TestDPAttentionDP2TP4VLM(CustomTestCase):
         kill_process_tree(cls.process.pid)
 
     def test_vlm_generate(self):
-        chat_template = get_chat_template_by_model_path(self.model)
+        # Go through /v1/chat/completions so the server inserts the model's own
-        prompt = f"{chat_template.image_token}What is in this image?"
+        # image placeholder instead of the test guessing one.
         response = requests.post(
-            self.base_url + "/generate",
+            self.base_url + "/v1/chat/completions",
             json={
-                "text": prompt,
+                "model": "default",
-                "image_data": [self.image_url],
+                "messages": [
-                "sampling_params": {
+                    {
-                    "temperature": 0,
+                        "role": "user",
-                    "max_new_tokens": 16,
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": self.image_url},
                             },
+                            {"type": "text", "text": "What is in this image?"},
+                        ],
+                    }
+                ],
+                "temperature": 0,
+                "max_tokens": 16,
             },
         )
         response.raise_for_status()
         response_json = response.json()
         print(response_json)
-        self.assertIn("output_ids", response_json)
+        self.assertTrue(response_json["choices"][0]["message"]["content"])
-        self.assertGreater(len(response_json["output_ids"]), 0)
+
+        # image_tokens comes from the prefill's multimodal item offsets, so a
+        # non-zero count is what proves the image reached the vision tower.
+        usage_details = response_json["usage"].get("prompt_tokens_details")
+        self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens")
+        self.assertGreater(usage_details.get("image_tokens", 0), 0)
 
 
 if __name__ == "__main__":

test/manual/quant/test_torchao.py

@@ -1,9 +1,9 @@
 import unittest
 
 import requests
+from transformers import AutoProcessor
 
 from sglang import Engine
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
 from sglang.test.kits.eval_accuracy_kit import MMLUMixin
 from sglang.test.test_utils import (
@@ -13,6 +13,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
+    build_vlm_image_prompt,
     is_in_amd_ci,
     popen_launch_server,
 )
@@ -72,8 +73,9 @@ class TestTorchAO(CustomTestCase, MMLUMixin):
 class TestTorchAOForVLM(CustomTestCase):
     def test_vlm_generate(self):
         model_path = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST
-        chat_template = get_chat_template_by_model_path(model_path)
+        text = build_vlm_image_prompt(
-        text = f"{chat_template.image_token}What is in this picture? Answer: "
+            AutoProcessor.from_pretrained(model_path), "What is in this picture?"
+        )
 
         engine = Engine(
             model_path=model_path,

test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py

@@ -1,9 +1,9 @@
 import unittest
 
 import torch
+from transformers import AutoProcessor
 
 from sglang import Engine
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.run_eval import run_eval
@@ -13,6 +13,7 @@ from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     SimpleNamespace,
+    build_vlm_image_prompt,
     is_in_amd_ci,
     popen_launch_server,
 )
@@ -73,8 +74,9 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase):
 
     def test_embedding(self):
         model_path = "Qwen/Qwen2.5-VL-3B-Instruct"
-        chat_template = get_chat_template_by_model_path(model_path)
+        text = build_vlm_image_prompt(
-        text = f"{chat_template.image_token}What is in this picture? Answer: "
+            AutoProcessor.from_pretrained(model_path), "What is in this picture?"
+        )
         extra_args = (
             {"mem_fraction_static": AMD_MEM_FRACTION_STATIC} if is_in_amd_ci() else {}
         )

test/registered/dp_attn/test_dp_attention.py

@@ -2,7 +2,6 @@ import unittest
 
 import requests
 
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
@@ -204,24 +203,38 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase):
         kill_process_tree(cls.process.pid)
 
     def test_vlm_generate(self):
-        chat_template = get_chat_template_by_model_path(self.model)
+        # Go through /v1/chat/completions so the server inserts the model's own
-        prompt = f"{chat_template.image_token}What is in this image?"
+        # image placeholder instead of the test guessing one.
         response = requests.post(
-            self.base_url + "/generate",
+            self.base_url + "/v1/chat/completions",
             json={
-                "text": prompt,
+                "model": "default",
-                "image_data": [self.image_url],
+                "messages": [
-                "sampling_params": {
+                    {
-                    "temperature": 0,
+                        "role": "user",
-                    "max_new_tokens": 16,
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": self.image_url},
                             },
+                            {"type": "text", "text": "What is in this image?"},
+                        ],
+                    }
+                ],
+                "temperature": 0,
+                "max_tokens": 16,
             },
         )
         response.raise_for_status()
         response_json = response.json()
         print(response_json)
-        self.assertIn("output_ids", response_json)
+        self.assertTrue(response_json["choices"][0]["message"]["content"])
-        self.assertGreater(len(response_json["output_ids"]), 0)
+
+        # image_tokens comes from the prefill's multimodal item offsets, so a
+        # non-zero count is what proves the image reached the vision tower.
+        usage_details = response_json["usage"].get("prompt_tokens_details")
+        self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens")
+        self.assertGreater(usage_details.get("image_tokens", 0), 0)
 
 
 if __name__ == "__main__":

test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py

@@ -2,7 +2,6 @@ import unittest
 
 import requests
 
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ascend.npu_eval_accuracy_kit import NPUGSM8KMixin
@@ -175,24 +174,38 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase):
         kill_process_tree(cls.process.pid)
 
     def test_vlm_generate(self):
-        chat_template = get_chat_template_by_model_path(self.model)
+        # Go through /v1/chat/completions so the server inserts the model's own
-        prompt = f"{chat_template.image_token}What is in this image?"
+        # image placeholder instead of the test guessing one.
         response = requests.post(
-            self.base_url + "/generate",
+            self.base_url + "/v1/chat/completions",
             json={
-                "text": prompt,
+                "model": "default",
-                "image_data": [self.image_url],
+                "messages": [
-                "sampling_params": {
+                    {
-                    "temperature": 0,
+                        "role": "user",
-                    "max_new_tokens": 16,
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": self.image_url},
                             },
+                            {"type": "text", "text": "What is in this image?"},
+                        ],
+                    }
+                ],
+                "temperature": 0,
+                "max_tokens": 16,
             },
         )
         response.raise_for_status()
         response_json = response.json()
         print(response_json)
-        self.assertIn("output_ids", response_json)
+        self.assertTrue(response_json["choices"][0]["message"]["content"])
-        self.assertGreater(len(response_json["output_ids"]), 0)
+
+        # image_tokens comes from the prefill's multimodal item offsets, so a
+        # non-zero count is what proves the image reached the vision tower.
+        usage_details = response_json["usage"].get("prompt_tokens_details")
+        self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens")
+        self.assertGreater(usage_details.get("image_tokens", 0), 0)
 
 
 if __name__ == "__main__":

test/registered/tokenizer/test_skip_tokenizer_init.py

@@ -9,7 +9,6 @@ import unittest
 import requests
 from transformers import AutoProcessor, AutoTokenizer
 
-from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.test_utils import (
@@ -19,6 +18,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
+    build_vlm_image_prompt,
     download_image_with_retry,
     popen_launch_server,
 )
@@ -91,9 +91,13 @@ class TestSkipTokenizerInit(CustomTestCase):
                 self.assertEqual(item["meta_info"]["prompt_tokens"], len(input_ids))
 
                 if return_logprob:
-                    num_input_logprobs = len(input_ids) - request["logprob_start_len"]
+                    # -1 resolves to the prompt end, so no input logprob is returned.
-                    if num_input_logprobs > len(input_ids):
+                    if request["logprob_start_len"] == -1:
-                        num_input_logprobs -= len(input_ids)
+                        num_input_logprobs = 0
+                    else:
+                        num_input_logprobs = (
+                            len(input_ids) - request["logprob_start_len"]
+                        )
                     self.assertEqual(
                         len(item["meta_info"]["input_token_logprobs"]),
                         num_input_logprobs,
@@ -230,8 +234,7 @@ class TestSkipTokenizerInitVLM(TestSkipTokenizerInit):
         cls.eos_token_id = [cls.tokenizer.eos_token_id]
 
     def get_input_ids(self, _prompt_text) -> list[int]:
-        chat_template = get_chat_template_by_model_path(self.model)
+        text = build_vlm_image_prompt(self.processor, "What is in this picture?")
-        text = f"{chat_template.image_token}What is in this picture?"
         inputs = self.processor(
             text=[text],
             images=[self.image],