vlm: refactor engine vlm params and support processor output as input (#14091)

Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: zhaochenyang20 <zhaochenyang20@gmail.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Co-authored-by: BenYao21 <cyao22@asu.edu> Co-authored-by: minleminzui <minleminzui@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: 赵晨阳 <zhaochen20@outlook.com>
2025-12-20 18:31:24 +08:00
parent 165f5c04cb
commit 1f1f05a85e
16 changed files with 783 additions and 305 deletions
@@ -219,12 +219,15 @@ class TestDeepseekOCRServer(TestOpenAIMLLMServerBase):
        self.verify_single_image_response_for_ocr(response)


+# Delete the mixin classes so that they are not collected by pytest
+del (
+    TestOpenAIMLLMServerBase,
+    ImageOpenAITestMixin,
+    VideoOpenAITestMixin,
+    AudioOpenAITestMixin,
+    OmniOpenAITestMixin,
+)
+
+
 if __name__ == "__main__":
-    del (
-        TestOpenAIMLLMServerBase,
-        ImageOpenAITestMixin,
-        VideoOpenAITestMixin,
-        AudioOpenAITestMixin,
-        OmniOpenAITestMixin,
-    )
    unittest.main()
@@ -1,20 +1,41 @@
 import json
 import unittest
+from io import BytesIO
 from typing import Optional

+import requests
 import torch
+
+# Compatibility shim: Kimi-VL dynamic module expects PytorchGELUTanh which may
+# be missing in transformers==4.57.1. Inject a lightweight implementation so
+# the model can import successfully without downgrading transformers.
+import transformers.activations as _hf_activations
+from PIL import Image
 from transformers import (
+    AutoModel,
    AutoProcessor,
    Gemma3ForConditionalGeneration,
    Qwen2_5_VLForConditionalGeneration,
 )

+if not hasattr(_hf_activations, "PytorchGELUTanh"):
+
+    class PytorchGELUTanh(torch.nn.Module):
+        def forward(self, x):
+            return torch.nn.functional.gelu(x, approximate="tanh")
+
+    _hf_activations.PytorchGELUTanh = PytorchGELUTanh
+    _hf_activations.ACT2FN.setdefault(
+        "pytorch_gelu_tanh",
+        lambda x: torch.nn.functional.gelu(x, approximate="tanh"),
+    )
+
 from sglang import Engine
 from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
 from sglang.srt.parser.conversation import generate_chat_conv
-from sglang.test.test_utils import download_image_with_retry

-TEST_IMAGE_URL = "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png"
+IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
+IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png"


 class VLMInputTestBase:
@@ -27,9 +48,12 @@ class VLMInputTestBase:
    def setUpClass(cls):
        assert cls.model_path is not None, "Set model_path in subclass"
        assert cls.chat_template is not None, "Set chat_template in subclass"
-        cls.image_url = TEST_IMAGE_URL
+        cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL]
        cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        cls.main_image = download_image_with_retry(cls.image_url)
+        cls.main_image = []
+        for image_url in cls.image_urls:
+            response = requests.get(image_url)
+            cls.main_image.append(Image.open(BytesIO(response.content)))
        cls.processor = AutoProcessor.from_pretrained(
            cls.model_path, trust_remote_code=True, use_fast=True
        )
@@ -55,8 +79,30 @@ class VLMInputTestBase:
        self.engine.shutdown()

    def verify_response(self, output):
+        # The goal is to check that the model roughly understands:
+        #   - image 1: taxi / car scene
+        #   - image 2: SGL logo / company
+        # We intentionally keep the check keyword-based and loose to avoid
+        # overfitting to a specific phrasing.
        out_text = output["text"].lower()
-        assert "taxi" in out_text or "cab" in out_text or "car" in out_text, out_text
+
+        assert any(w in out_text for w in ("taxi", "cab", "car")), out_text
+
+        has_sg_or_logo_side = any(
+            kw in out_text
+            for kw in (
+                "sg ",
+                "sgl",
+                " sgl",
+                "logo",
+                "software guidance",
+                "labs",
+                "laborator",
+                "company",
+                " text",
+            )
+        )
+        assert has_sg_or_logo_side, out_text

    def get_completion_request(self) -> ChatCompletionRequest:
        json_structure = {
@@ -65,8 +111,12 @@ class VLMInputTestBase:
                {
                    "role": "user",
                    "content": [
-                        {"type": "image_url", "image_url": {"url": self.image_url}},
-                        {"type": "text", "text": "What's in this picture?"},
+                        {"type": "image_url", "image_url": {"url": self.image_urls[0]}},
+                        {"type": "image_url", "image_url": {"url": self.image_urls[1]}},
+                        {
+                            "type": "text",
+                            "text": "Describe both the first image and the second image in detail separately.",  # update prompt, ensure kimi-vl understands the images separately.
+                        },
                    ],
                }
            ],
@@ -83,55 +133,58 @@ class VLMInputTestBase:
        # Process inputs using processor
        inputs = self.processor(
            text=[text],
-            images=[self.main_image],
+            images=self.main_image,
            return_tensors="pt",
        ).to(self.device)

-        return inputs
+        return inputs, text

-    async def test_understands_image(self):
+    async def test_accepts_image(self):
        req = self.get_completion_request()
        conv = generate_chat_conv(req, template_name=self.chat_template)
        text = conv.get_prompt()
        output = await self.engine.async_generate(
            prompt=text,
-            image_data=[self.main_image],
-            sampling_params=dict(temperature=0.0),
+            image_data=self.main_image,
+            sampling_params=dict(temperature=0.0, max_new_tokens=512),
        )
        self.verify_response(output)

-    async def test_understands_precomputed_embeddings(self):
+    async def test_accepts_precomputed_embeddings(self):
        req = self.get_completion_request()
-        processor_output = self.get_processor_output(req=req)
+        processor_output, _ = self.get_processor_output(req=req)
+
        with torch.inference_mode():
            precomputed_embeddings = self.__class__.visual(processor_output)
+
        output = await self.engine.async_generate(
            input_ids=processor_output["input_ids"][0].detach().cpu().tolist(),
            image_data=[
                self._precomputed_image_data(processor_output, precomputed_embeddings)
            ],
-            sampling_params=dict(temperature=0.0),
+            sampling_params=dict(temperature=0.0, max_new_tokens=512),
        )
        self.verify_response(output)

-    async def test_understands_pixel_values(self):
+    async def test_accepts_processor_output(self):
        req = self.get_completion_request()
-        processor_output = self.get_processor_output(req=req)
+        processor_output, prompt = self.get_processor_output(req=req)
        output = await self.engine.async_generate(
            input_ids=processor_output["input_ids"][0].detach().cpu().tolist(),
-            image_data=[self._pixel_values_image_data(processor_output)],
-            sampling_params=dict(temperature=0.0),
+            image_data=[self._processor_output_image_data(processor_output)],
+            sampling_params=dict(temperature=0.0, max_new_tokens=512),
        )
        self.verify_response(output)

    def _precomputed_image_data(self, processor_output, precomputed_embeddings):
        """This should not be overridden."""
        return dict(
-            modality="IMAGE",
-            precomputed_embeddings=precomputed_embeddings,
+            processor_output,
+            format="precomputed_embedding",
+            feature=precomputed_embeddings,
        )

-    def _pixel_values_image_data(self, processor_output):
+    def _processor_output_image_data(self, processor_output):
        """Override in subclass to pass the correct set of arguments."""
        raise NotImplementedError

@@ -153,12 +206,8 @@ class TestQwenVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestC
            processor_output["pixel_values"], processor_output["image_grid_thw"]
        )

-    def _pixel_values_image_data(self, processor_output):
-        return dict(
-            modality="IMAGE",
-            image_grid_thw=processor_output["image_grid_thw"],
-            pixel_values=processor_output["pixel_values"],
-        )
+    def _processor_output_image_data(self, processor_output):
+        return dict(processor_output, format="processor_output")


 class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
@@ -170,57 +219,60 @@ class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCa
        model = Gemma3ForConditionalGeneration.from_pretrained(
            cls.model_path, torch_dtype=torch.bfloat16
        )
-        cls.vision_tower = model.vision_tower.eval().to(cls.device)
-        cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
+        base_model = model.model
+
+        cls.vision_tower = base_model.vision_tower.eval().to(cls.device)
+
+        if hasattr(base_model, "multi_modal_projector"):
+            cls.mm_projector = base_model.multi_modal_projector.eval().to(cls.device)
+        else:
+            cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
+
        cls.visual = lambda processor_output: cls.mm_projector(
            cls.vision_tower(
                pixel_values=processor_output["pixel_values"]
            ).last_hidden_state
        )

-    def _pixel_values_image_data(self, processor_output):
-        return dict(
-            modality="IMAGE",
-            pixel_values=processor_output["pixel_values"][0],
+    def _processor_output_image_data(self, processor_output):
+        return dict(processor_output, format="processor_output")
+
+
+# Updated Kimi-VL test to use the new input format.
+class TestKimiVLImageUnderstandsImage(
+    VLMInputTestBase, unittest.IsolatedAsyncioTestCase
+):
+    model_path = "moonshotai/Kimi-VL-A3B-Instruct"
+    chat_template = "kimi-vl"
+
+    @classmethod
+    def _init_visual(cls):
+        model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True)
+        cls.vision_tower = model.vision_tower.eval().to(cls.device)
+        cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
+
+        cls.visual = lambda tokenizer_output: cls.mm_projector(
+            cls.vision_tower(
+                pixel_values=tokenizer_output["pixel_values"],
+                grid_hws=tokenizer_output["image_grid_hws"],
+            )
        )

-
-# Temporarily skip Kimi-VL for CI test due to issue in transformers=4.57.0
-# class TestKimiVLImageUnderstandsImage(
-#     VLMInputTestBase, unittest.IsolatedAsyncioTestCase
-# ):
-#     model_path = "moonshotai/Kimi-VL-A3B-Instruct"
-#     chat_template = "kimi-vl"
-
-#     @classmethod
-#     def _init_visual(cls):
-#         model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True)
-#         cls.vision_tower = model.vision_tower.eval().to(cls.device)
-#         cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
-
-#         cls.visual = lambda tokenizer_output: cls.mm_projector(
-#             cls.vision_tower(
-#                 pixel_values=tokenizer_output["pixel_values"],
-#                 grid_hws=tokenizer_output["image_grid_hws"],
-#             )
-#         )
-
-#     def _pixel_values_image_data(self, processor_output):
-#         return dict(
-#             modality="IMAGE",
-#             pixel_values=processor_output["pixel_values"],
-#             image_grid_hws=processor_output["image_grid_hws"],
-#         )
+    def _processor_output_image_data(self, processor_output):
+        return dict(processor_output, format="processor_output")


 # not for CI: too large
 # class TestLlama4ImageUnderstandsImage(
 #     VLMInputTestBase, unittest.IsolatedAsyncioTestCase
 # ):
+#     # Allow overriding via env for local/offline runs.
 #     model_path = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
-#     chat_template = "llama_4_vision"
+#     chat_template = "llama-4"

 #     def setUp(self):
+#         if torch.cuda.device_count() < 4:
+#             self.skipTest("Skipping Llama-4 test: requires 4 GPUs for TP=4")
 #         self.engine = Engine(
 #             model_path=self.model_path,
 #             trust_remote_code=True,
@@ -234,7 +286,12 @@ class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCa

 #     @classmethod
 #     def _init_visual(cls):
-#         model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True, torch_dtype="auto")
+#         model = AutoModel.from_pretrained(
+#             cls.model_path,
+#             trust_remote_code=True,
+#             torch_dtype="auto",
+#             force_download=True,
+#         )
 #         cls.vision_tower = model.vision_model.eval().to(cls.device)
 #         cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)

@@ -244,11 +301,48 @@ class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCa
 #             ).last_hidden_state.flatten(0, -2)
 #         )

-#     def _pixel_values_image_data(self, processor_output):
-#         return dict(
-#             modality="IMAGE",
-#             pixel_values=processor_output["pixel_values"],
+#     def _processor_output_image_data(self, processor_output):
+#         # Llama-4 vision expects processor_output format with pixel_values
+#         return dict(processor_output, format="processor_output")
+
+
+# class TestLlavaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
+#     model_path = "llava-hf/llava-1.5-7b-hf"
+#     chat_template = "vicuna_v1.1"
+
+#     @classmethod
+#     def _init_visual(cls):
+#         from transformers import LlavaForConditionalGeneration
+
+#         model = LlavaForConditionalGeneration.from_pretrained(
+#             cls.model_path,
+#             torch_dtype=torch.float16,
+#             low_cpu_mem_usage=True,
 #         )
+#         cls.vision_tower = model.vision_tower.eval().to(cls.device)
+#         cls.multi_modal_projector = model.multi_modal_projector.eval().to(cls.device)
+#         cls.config = model.config
+
+#         def visual_func(processor_output):
+#             pixel_values = processor_output["pixel_values"].to(
+#                 cls.device, dtype=torch.float16
+#             )
+
+#             vision_outputs = cls.vision_tower(pixel_values, output_hidden_states=True)
+#             image_features = vision_outputs.hidden_states[-2]
+
+#             if cls.config.vision_feature_select_strategy == "default":
+#                 image_features = image_features[:, 1:]
+#             elif cls.config.vision_feature_select_strategy == "full":
+#                 image_features = image_features
+
+#             image_features = cls.multi_modal_projector(image_features)
+#             return image_features
+
+#         cls.visual = visual_func
+
+#     def _processor_output_image_data(self, processor_output):
+#         return dict(processor_output, format="processor_output")


 if __name__ == "__main__":