diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index aa7535ca6..efd41ed4e 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -183,6 +183,22 @@ def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Ima time.sleep(2**i) +def build_vlm_image_prompt(processor, question: str) -> str: + # Take the image placeholder from the model's own HF chat template: a + # hand-written one silently degrades to a text-only prompt on any model + # whose placeholder differs. + return processor.apply_chat_template( + [ + { + "role": "user", + "content": [{"type": "image"}, {"type": "text", "text": question}], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + + def is_in_ci(): """Return whether it is in CI runner.""" return get_bool_env_var("SGLANG_IS_IN_CI") diff --git a/test/manual/distributed/test_dp_attention_large.py b/test/manual/distributed/test_dp_attention_large.py index 561ca06fc..997491a1c 100644 --- a/test/manual/distributed/test_dp_attention_large.py +++ b/test/manual/distributed/test_dp_attention_large.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin @@ -164,24 +163,38 @@ class TestDPAttentionDP2TP4VLM(CustomTestCase): kill_process_tree(cls.process.pid) def test_vlm_generate(self): - chat_template = get_chat_template_by_model_path(self.model) - prompt = f"{chat_template.image_token}What is in this image?" + # Go through /v1/chat/completions so the server inserts the model's own + # image placeholder instead of the test guessing one. response = requests.post( - self.base_url + "/generate", + self.base_url + "/v1/chat/completions", json={ - "text": prompt, - "image_data": [self.image_url], - "sampling_params": { - "temperature": 0, - "max_new_tokens": 16, - }, + "model": "default", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.image_url}, + }, + {"type": "text", "text": "What is in this image?"}, + ], + } + ], + "temperature": 0, + "max_tokens": 16, }, ) response.raise_for_status() response_json = response.json() print(response_json) - self.assertIn("output_ids", response_json) - self.assertGreater(len(response_json["output_ids"]), 0) + self.assertTrue(response_json["choices"][0]["message"]["content"]) + + # image_tokens comes from the prefill's multimodal item offsets, so a + # non-zero count is what proves the image reached the vision tower. + usage_details = response_json["usage"].get("prompt_tokens_details") + self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens") + self.assertGreater(usage_details.get("image_tokens", 0), 0) if __name__ == "__main__": diff --git a/test/manual/quant/test_torchao.py b/test/manual/quant/test_torchao.py index 37006b866..69ccf66d1 100644 --- a/test/manual/quant/test_torchao.py +++ b/test/manual/quant/test_torchao.py @@ -1,9 +1,9 @@ import unittest import requests +from transformers import AutoProcessor from sglang import Engine -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( @@ -13,6 +13,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + build_vlm_image_prompt, is_in_amd_ci, popen_launch_server, ) @@ -72,8 +73,9 @@ class TestTorchAO(CustomTestCase, MMLUMixin): class TestTorchAOForVLM(CustomTestCase): def test_vlm_generate(self): model_path = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST - chat_template = get_chat_template_by_model_path(model_path) - text = f"{chat_template.image_token}What is in this picture? Answer: " + text = build_vlm_image_prompt( + AutoProcessor.from_pretrained(model_path), "What is in this picture?" + ) engine = Engine( model_path=model_path, diff --git a/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py b/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py index 4ec0637d6..6f7742a75 100644 --- a/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py +++ b/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py @@ -1,9 +1,9 @@ import unittest import torch +from transformers import AutoProcessor from sglang import Engine -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval @@ -13,6 +13,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, SimpleNamespace, + build_vlm_image_prompt, is_in_amd_ci, popen_launch_server, ) @@ -73,8 +74,9 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): def test_embedding(self): model_path = "Qwen/Qwen2.5-VL-3B-Instruct" - chat_template = get_chat_template_by_model_path(model_path) - text = f"{chat_template.image_token}What is in this picture? Answer: " + text = build_vlm_image_prompt( + AutoProcessor.from_pretrained(model_path), "What is in this picture?" + ) extra_args = ( {"mem_fraction_static": AMD_MEM_FRACTION_STATIC} if is_in_amd_ci() else {} ) diff --git a/test/registered/dp_attn/test_dp_attention.py b/test/registered/dp_attn/test_dp_attention.py index 95d34e76b..1fb2dcb66 100644 --- a/test/registered/dp_attn/test_dp_attention.py +++ b/test/registered/dp_attn/test_dp_attention.py @@ -2,7 +2,6 @@ import unittest import requests -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -204,24 +203,38 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase): kill_process_tree(cls.process.pid) def test_vlm_generate(self): - chat_template = get_chat_template_by_model_path(self.model) - prompt = f"{chat_template.image_token}What is in this image?" + # Go through /v1/chat/completions so the server inserts the model's own + # image placeholder instead of the test guessing one. response = requests.post( - self.base_url + "/generate", + self.base_url + "/v1/chat/completions", json={ - "text": prompt, - "image_data": [self.image_url], - "sampling_params": { - "temperature": 0, - "max_new_tokens": 16, - }, + "model": "default", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.image_url}, + }, + {"type": "text", "text": "What is in this image?"}, + ], + } + ], + "temperature": 0, + "max_tokens": 16, }, ) response.raise_for_status() response_json = response.json() print(response_json) - self.assertIn("output_ids", response_json) - self.assertGreater(len(response_json["output_ids"]), 0) + self.assertTrue(response_json["choices"][0]["message"]["content"]) + + # image_tokens comes from the prefill's multimodal item offsets, so a + # non-zero count is what proves the image reached the vision tower. + usage_details = response_json["usage"].get("prompt_tokens_details") + self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens") + self.assertGreater(usage_details.get("image_tokens", 0), 0) if __name__ == "__main__": diff --git a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py index 4b8cfde37..aed6429a8 100644 --- a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py +++ b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py @@ -2,7 +2,6 @@ import unittest import requests -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ascend.npu_eval_accuracy_kit import NPUGSM8KMixin @@ -175,24 +174,38 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase): kill_process_tree(cls.process.pid) def test_vlm_generate(self): - chat_template = get_chat_template_by_model_path(self.model) - prompt = f"{chat_template.image_token}What is in this image?" + # Go through /v1/chat/completions so the server inserts the model's own + # image placeholder instead of the test guessing one. response = requests.post( - self.base_url + "/generate", + self.base_url + "/v1/chat/completions", json={ - "text": prompt, - "image_data": [self.image_url], - "sampling_params": { - "temperature": 0, - "max_new_tokens": 16, - }, + "model": "default", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.image_url}, + }, + {"type": "text", "text": "What is in this image?"}, + ], + } + ], + "temperature": 0, + "max_tokens": 16, }, ) response.raise_for_status() response_json = response.json() print(response_json) - self.assertIn("output_ids", response_json) - self.assertGreater(len(response_json["output_ids"]), 0) + self.assertTrue(response_json["choices"][0]["message"]["content"]) + + # image_tokens comes from the prefill's multimodal item offsets, so a + # non-zero count is what proves the image reached the vision tower. + usage_details = response_json["usage"].get("prompt_tokens_details") + self.assertIsNotNone(usage_details, "prompt carried no multimodal tokens") + self.assertGreater(usage_details.get("image_tokens", 0), 0) if __name__ == "__main__": diff --git a/test/registered/tokenizer/test_skip_tokenizer_init.py b/test/registered/tokenizer/test_skip_tokenizer_init.py index babf1dec5..2b93f380a 100644 --- a/test/registered/tokenizer/test_skip_tokenizer_init.py +++ b/test/registered/tokenizer/test_skip_tokenizer_init.py @@ -9,7 +9,6 @@ import unittest import requests from transformers import AutoProcessor, AutoTokenizer -from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( @@ -19,6 +18,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + build_vlm_image_prompt, download_image_with_retry, popen_launch_server, ) @@ -91,9 +91,13 @@ class TestSkipTokenizerInit(CustomTestCase): self.assertEqual(item["meta_info"]["prompt_tokens"], len(input_ids)) if return_logprob: - num_input_logprobs = len(input_ids) - request["logprob_start_len"] - if num_input_logprobs > len(input_ids): - num_input_logprobs -= len(input_ids) + # -1 resolves to the prompt end, so no input logprob is returned. + if request["logprob_start_len"] == -1: + num_input_logprobs = 0 + else: + num_input_logprobs = ( + len(input_ids) - request["logprob_start_len"] + ) self.assertEqual( len(item["meta_info"]["input_token_logprobs"]), num_input_logprobs, @@ -230,8 +234,7 @@ class TestSkipTokenizerInitVLM(TestSkipTokenizerInit): cls.eos_token_id = [cls.tokenizer.eos_token_id] def get_input_ids(self, _prompt_text) -> list[int]: - chat_template = get_chat_template_by_model_path(self.model) - text = f"{chat_template.image_token}What is in this picture?" + text = build_vlm_image_prompt(self.processor, "What is in this picture?") inputs = self.processor( text=[text], images=[self.image],