ci: migrate VLM tests to test/registered/vlm/ (#16415)
This commit is contained in:
Alison Shao
@@ -0,0 +1,253 @@
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=150, suite="stage-b-test-small-1-gpu")
|
||||
|
||||
"""
|
||||
Usage:
|
||||
python3 -m unittest test_vision_chunked_prefill.TestVisionChunkedPrefill.test_chunked_prefill
|
||||
"""
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Union
|
||||
|
||||
import numpy as np
|
||||
import pybase64
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
calculate_rouge_l,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
# Configure logging to help diagnose CI timeouts
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TestVisionChunkedPrefill(CustomTestCase):
|
||||
|
||||
def prepare_video_messages(self, video_path, max_frames_num=8):
|
||||
# We import decord here to avoid a strange Segmentation fault (core dumped) issue.
|
||||
# The following import order will cause Segmentation fault.
|
||||
# import decord
|
||||
# from transformers import AutoTokenizer
|
||||
from decord import VideoReader, cpu
|
||||
|
||||
vr = VideoReader(video_path, ctx=cpu(0))
|
||||
total_frame_num = len(vr)
|
||||
uniform_sampled_frames = np.linspace(
|
||||
0, total_frame_num - 1, max_frames_num, dtype=int
|
||||
)
|
||||
frame_idx = uniform_sampled_frames.tolist()
|
||||
frames = vr.get_batch(frame_idx).asnumpy()
|
||||
|
||||
base64_frames = []
|
||||
for frame in frames:
|
||||
pil_img = Image.fromarray(frame)
|
||||
buff = io.BytesIO()
|
||||
pil_img.save(buff, format="JPEG")
|
||||
base64_str = pybase64.b64encode(buff.getvalue()).decode("utf-8")
|
||||
base64_frames.append(base64_str)
|
||||
|
||||
messages = [{"role": "user", "content": []}]
|
||||
frame_format = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,{}"},
|
||||
"modalities": "video",
|
||||
}
|
||||
|
||||
for base64_frame in base64_frames:
|
||||
frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format(
|
||||
base64_frame
|
||||
)
|
||||
messages[0]["content"].append(frame_format.copy())
|
||||
|
||||
prompt = {"type": "text", "text": "Please describe the video briefly."}
|
||||
messages[0]["content"].append(prompt)
|
||||
|
||||
return messages
|
||||
|
||||
def get_prompt_from_messages(self, messages):
|
||||
text = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
)
|
||||
image_data = []
|
||||
for content in messages[0]["content"]:
|
||||
if content["type"] == "image_url":
|
||||
text += "<image>\n"
|
||||
image_data.append(content["image_url"]["url"])
|
||||
text += "Please describe the video briefly.<|im_end|>\n<|im_start|>assistant\n"
|
||||
return text, image_data
|
||||
|
||||
def generate(self, text, image_data):
|
||||
num_images = len(image_data) if image_data else 0
|
||||
logger.info(f"Starting generate request with {num_images} images")
|
||||
start_time = time.time()
|
||||
response = requests.post(
|
||||
self.base_url + "/generate",
|
||||
json={
|
||||
"text": text,
|
||||
"image_data": image_data,
|
||||
"sampling_params": {
|
||||
"temperature": 0,
|
||||
"max_new_tokens": 32,
|
||||
"no_stop_trim": True,
|
||||
"skip_special_tokens": False,
|
||||
},
|
||||
"modalities": ["multi-images"],
|
||||
},
|
||||
timeout=120, # Add timeout to prevent hanging indefinitely
|
||||
).json()
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"Generate request completed in {elapsed:.2f}s")
|
||||
return response["text"]
|
||||
|
||||
def generate_for_video(self, batch, num_frame) -> Union[str, list[str]]:
|
||||
logger.info(
|
||||
f"generate_for_video called with batch={batch}, num_frame={num_frame}"
|
||||
)
|
||||
|
||||
# prepare the video input about Steven introducing ipod nano
|
||||
url = "https://raw.githubusercontent.com/evolvinglmms-lab/sglang/dev/onevision_local/assets/jobs.mp4"
|
||||
cache_dir = os.path.expanduser("~/.cache")
|
||||
file_path = os.path.join(cache_dir, "jobs.mp4")
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
if not os.path.exists(file_path):
|
||||
logger.info(f"Downloading video from {url}")
|
||||
start_time = time.time()
|
||||
response = requests.get(url, timeout=60)
|
||||
response.raise_for_status()
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
f"Video downloaded in {elapsed:.2f}s, size={len(response.content)} bytes"
|
||||
)
|
||||
else:
|
||||
logger.info(f"Using cached video at {file_path}")
|
||||
|
||||
if not batch:
|
||||
assert isinstance(num_frame, int)
|
||||
logger.info(f"Processing single video with {num_frame} frames")
|
||||
messages = self.prepare_video_messages(file_path, max_frames_num=num_frame)
|
||||
text, image_data = self.get_prompt_from_messages(messages)
|
||||
return self.generate(text, image_data)
|
||||
else:
|
||||
assert isinstance(num_frame, list)
|
||||
logger.info(f"Processing batch of videos with frame counts: {num_frame}")
|
||||
func_args = []
|
||||
for max_frames_num in num_frame:
|
||||
messages = self.prepare_video_messages(
|
||||
file_path,
|
||||
max_frames_num=max_frames_num,
|
||||
)
|
||||
text, image_data = self.get_prompt_from_messages(messages)
|
||||
func_args.append((text, image_data))
|
||||
|
||||
logger.info(f"Starting batch generation with {len(func_args)} requests")
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
responses = list(executor.map(lambda p: self.generate(*p), func_args))
|
||||
logger.info(f"Batch generation completed")
|
||||
|
||||
return responses
|
||||
|
||||
def launch_server(self, chunked_prefill_size) -> int:
|
||||
# launch server
|
||||
model = "lmms-lab/llava-onevision-qwen2-7b-ov"
|
||||
# model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
||||
self.base_url = DEFAULT_URL_FOR_TEST
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
self.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=[
|
||||
"--chunked-prefill-size",
|
||||
f"{chunked_prefill_size}",
|
||||
],
|
||||
)
|
||||
return process.pid
|
||||
|
||||
def _test_chunked_prefill(self, batches, num_frames):
|
||||
logger.info("=" * 60)
|
||||
logger.info("Starting chunked prefill test")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Chunked
|
||||
logger.info("Phase 1: Testing with chunked_prefill_size=1024")
|
||||
chunked_server_pid = self.launch_server(chunked_prefill_size=1024)
|
||||
logger.info(f"Chunked server started with pid={chunked_server_pid}")
|
||||
try:
|
||||
outputs_chunked = []
|
||||
for i, (batch, num_frame) in enumerate(zip(batches, num_frames)):
|
||||
logger.info(f"Chunked test iteration {i+1}/{len(batches)}")
|
||||
output_chunked = self.generate_for_video(
|
||||
batch=batch, num_frame=num_frame
|
||||
)
|
||||
outputs_chunked += [output_chunked]
|
||||
logger.info(f"Chunked test iteration {i+1} completed")
|
||||
finally:
|
||||
logger.info(f"Killing chunked server pid={chunked_server_pid}")
|
||||
kill_process_tree(chunked_server_pid)
|
||||
logger.info("Chunked server killed")
|
||||
time.sleep(4)
|
||||
|
||||
# None-chunked
|
||||
logger.info("Phase 2: Testing with chunked_prefill_size=-1 (no chunking)")
|
||||
try:
|
||||
no_chunked_server_pid = self.launch_server(chunked_prefill_size=-1)
|
||||
logger.info(f"Non-chunked server started with pid={no_chunked_server_pid}")
|
||||
outputs_no_chunked = []
|
||||
for i, (batch, num_frame) in enumerate(zip(batches, num_frames)):
|
||||
logger.info(f"Non-chunked test iteration {i+1}/{len(batches)}")
|
||||
output_no_chunked = self.generate_for_video(
|
||||
batch=batch, num_frame=num_frame
|
||||
)
|
||||
outputs_no_chunked += [output_no_chunked]
|
||||
logger.info(f"Non-chunked test iteration {i+1} completed")
|
||||
|
||||
finally:
|
||||
logger.info(f"Killing non-chunked server pid={no_chunked_server_pid}")
|
||||
kill_process_tree(no_chunked_server_pid)
|
||||
logger.info("Non-chunked server killed")
|
||||
time.sleep(4)
|
||||
|
||||
for output_chunked, output_no_chunked in zip(
|
||||
outputs_chunked, outputs_no_chunked
|
||||
):
|
||||
print("output with chunked prefill:")
|
||||
print(output_chunked)
|
||||
print("output without chunked prefill:")
|
||||
print(output_no_chunked)
|
||||
self.assertEqual(len(output_chunked), len(output_no_chunked))
|
||||
rouge_scores = calculate_rouge_l(output_chunked, output_no_chunked)
|
||||
avg_score = sum(rouge_scores) / len(rouge_scores)
|
||||
print(f"ROUGE-L scores: {rouge_scores}")
|
||||
print(f"Average ROUGE-L score: {avg_score:.4f}")
|
||||
# Allow for occasional divergence in one item while maintaining overall output quality
|
||||
self.assertGreater(
|
||||
avg_score,
|
||||
0.90,
|
||||
f"Average ROUGE-L score too low: {avg_score:.4f}. "
|
||||
f"Individual scores: {rouge_scores}",
|
||||
)
|
||||
|
||||
def test_chunked_prefill(self):
|
||||
self._test_chunked_prefill(batches=[False, True], num_frames=[1, [2, 6, 8, 10]])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,238 @@
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=890, suite="stage-b-test-small-1-gpu")
|
||||
|
||||
"""
|
||||
Usage:
|
||||
python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_mixed_batch
|
||||
python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_multi_images_chat_completion
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.test.vlm_utils import *
|
||||
from sglang.test.vlm_utils import (
|
||||
AudioOpenAITestMixin,
|
||||
CustomTestCase,
|
||||
ImageOpenAITestMixin,
|
||||
OmniOpenAITestMixin,
|
||||
TestOpenAIMLLMServerBase,
|
||||
VideoOpenAITestMixin,
|
||||
)
|
||||
|
||||
|
||||
class TestLlavaServer(ImageOpenAITestMixin):
|
||||
model = "lmms-lab/llava-onevision-qwen2-0.5b-ov"
|
||||
|
||||
|
||||
class TestQwen25VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
|
||||
model = "Qwen/Qwen2.5-VL-7B-Instruct"
|
||||
extra_args = [
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
|
||||
class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
|
||||
model = "Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
extra_args = ["--cuda-graph-max-bs=4"]
|
||||
|
||||
|
||||
class TestQwen3OmniServer(OmniOpenAITestMixin):
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
extra_args = [ # workaround to fit into H100
|
||||
"--mem-fraction-static=0.90",
|
||||
"--disable-cuda-graph",
|
||||
"--disable-fast-image-processor",
|
||||
"--grammar-backend=none",
|
||||
]
|
||||
|
||||
|
||||
class TestQwen2VLContextLengthServer(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "Qwen/Qwen2-VL-7B-Instruct"
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
api_key=cls.api_key,
|
||||
other_args=[
|
||||
"--context-length",
|
||||
"300",
|
||||
"--cuda-graph-max-bs",
|
||||
"4",
|
||||
],
|
||||
)
|
||||
cls.base_url += "/v1"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_single_image_chat_completion(self):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
with self.assertRaises(openai.BadRequestError) as cm:
|
||||
client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_MAN_IRONING_URL},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Give a lengthy description of this picture",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
# context length is checked first, then max_req_input_len, which is calculated from the former
|
||||
assert (
|
||||
"Multimodal prompt is too long after expanding multimodal tokens."
|
||||
in str(cm.exception)
|
||||
or "is longer than the model's context length" in str(cm.exception)
|
||||
)
|
||||
|
||||
|
||||
# flaky
|
||||
# class TestMllamaServer(ImageOpenAITestMixin):
|
||||
# model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
||||
|
||||
|
||||
class TestInternVL25Server(ImageOpenAITestMixin):
|
||||
model = "OpenGVLab/InternVL2_5-2B"
|
||||
extra_args = [
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
|
||||
class TestMiniCPMV4Server(ImageOpenAITestMixin):
|
||||
model = "openbmb/MiniCPM-V-4"
|
||||
extra_args = [
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
|
||||
class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin):
|
||||
model = "openbmb/MiniCPM-o-2_6"
|
||||
extra_args = [
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
|
||||
class TestGemma3itServer(ImageOpenAITestMixin):
|
||||
model = "google/gemma-3-4b-it"
|
||||
extra_args = [
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
|
||||
class TestKimiVLServer(ImageOpenAITestMixin):
|
||||
model = "moonshotai/Kimi-VL-A3B-Instruct"
|
||||
extra_args = [
|
||||
"--context-length=8192",
|
||||
"--dtype=bfloat16",
|
||||
]
|
||||
|
||||
def test_video_images_chat_completion(self):
|
||||
# model context length exceeded
|
||||
pass
|
||||
|
||||
|
||||
@unittest.skip(
|
||||
"Disabling this test to speed up CI. Prefer to test it within nightly test."
|
||||
)
|
||||
class TestGLM41VServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
|
||||
model = "zai-org/GLM-4.1V-9B-Thinking"
|
||||
extra_args = [
|
||||
"--reasoning-parser=glm45",
|
||||
]
|
||||
|
||||
|
||||
class TestQwen2AudioServer(AudioOpenAITestMixin):
|
||||
model = "Qwen/Qwen2-Audio-7B-Instruct"
|
||||
|
||||
|
||||
class TestDeepseekOCRServer(TestOpenAIMLLMServerBase):
|
||||
model = "deepseek-ai/DeepSeek-OCR"
|
||||
trust_remote_code = False
|
||||
extra_args = [
|
||||
"--mem-fraction-static=0.70",
|
||||
"--cuda-graph-max-bs=4",
|
||||
]
|
||||
|
||||
def verify_single_image_response_for_ocr(self, response):
|
||||
"""Verify DeepSeek-OCR grounding output with coordinates"""
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
text = response.choices[0].message.content
|
||||
assert isinstance(text, str)
|
||||
|
||||
# DeepSeek-OCR uses grounding format, outputs coordinates
|
||||
assert "text" in text.lower(), f"OCR text: {text}, should contain 'text'"
|
||||
|
||||
# Verify coordinate format [[x1, y1, x2, y2]]
|
||||
import re
|
||||
|
||||
coord_pattern = r"\[\[[\d\s,]+\]\]"
|
||||
assert re.search(
|
||||
coord_pattern, text
|
||||
), f"OCR text: {text}, should contain coordinate format [[x1, y1, x2, y2]]"
|
||||
|
||||
# Verify basic response fields
|
||||
assert response.id
|
||||
assert response.created
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens > 0
|
||||
|
||||
def test_single_image_chat_completion(self):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
image_url = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/ocr-text.png"
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "<|grounding|>Convert the document to markdown.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
**(self.get_vision_request_kwargs()),
|
||||
)
|
||||
|
||||
self.verify_single_image_response_for_ocr(response)
|
||||
|
||||
|
||||
# Delete the mixin classes so that they are not collected by pytest
|
||||
del (
|
||||
TestOpenAIMLLMServerBase,
|
||||
ImageOpenAITestMixin,
|
||||
VideoOpenAITestMixin,
|
||||
AudioOpenAITestMixin,
|
||||
OmniOpenAITestMixin,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,353 @@
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=430, suite="stage-b-test-small-1-gpu")
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
import torch
|
||||
|
||||
# Compatibility shim: Kimi-VL dynamic module expects PytorchGELUTanh which may
|
||||
# be missing in transformers==4.57.1. Inject a lightweight implementation so
|
||||
# the model can import successfully without downgrading transformers.
|
||||
import transformers.activations as _hf_activations
|
||||
from PIL import Image
|
||||
from transformers import (
|
||||
AutoModel,
|
||||
AutoProcessor,
|
||||
Gemma3ForConditionalGeneration,
|
||||
Qwen2_5_VLForConditionalGeneration,
|
||||
)
|
||||
|
||||
if not hasattr(_hf_activations, "PytorchGELUTanh"):
|
||||
|
||||
class PytorchGELUTanh(torch.nn.Module):
|
||||
def forward(self, x):
|
||||
return torch.nn.functional.gelu(x, approximate="tanh")
|
||||
|
||||
_hf_activations.PytorchGELUTanh = PytorchGELUTanh
|
||||
_hf_activations.ACT2FN.setdefault(
|
||||
"pytorch_gelu_tanh",
|
||||
lambda x: torch.nn.functional.gelu(x, approximate="tanh"),
|
||||
)
|
||||
|
||||
from sglang import Engine
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.parser.conversation import generate_chat_conv
|
||||
|
||||
IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
|
||||
IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png"
|
||||
|
||||
|
||||
class VLMInputTestBase:
|
||||
model_path = None
|
||||
chat_template = None
|
||||
processor = None
|
||||
visual = None # Should be a callable for precomputed embeddings
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
assert cls.model_path is not None, "Set model_path in subclass"
|
||||
assert cls.chat_template is not None, "Set chat_template in subclass"
|
||||
cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL]
|
||||
cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
cls.main_image = []
|
||||
for image_url in cls.image_urls:
|
||||
response = requests.get(image_url)
|
||||
cls.main_image.append(Image.open(BytesIO(response.content)))
|
||||
cls.processor = AutoProcessor.from_pretrained(
|
||||
cls.model_path, trust_remote_code=True, use_fast=True
|
||||
)
|
||||
cls._init_visual()
|
||||
|
||||
@classmethod
|
||||
def _init_visual(cls):
|
||||
"""Override in subclass to set up cls.visual as a callable for precomputed embeddings."""
|
||||
raise NotImplementedError
|
||||
|
||||
def setUp(self):
|
||||
self.engine = Engine(
|
||||
model_path=self.model_path,
|
||||
chat_template=self.chat_template,
|
||||
device=self.device.type,
|
||||
mem_fraction_static=0.8,
|
||||
enable_multimodal=True,
|
||||
disable_cuda_graph=True,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
self.engine.shutdown()
|
||||
|
||||
def verify_response(self, output):
|
||||
# The goal is to check that the model roughly understands:
|
||||
# - image 1: taxi / car scene
|
||||
# - image 2: SGL logo / company
|
||||
# We intentionally keep the check keyword-based and loose to avoid
|
||||
# overfitting to a specific phrasing.
|
||||
out_text = output["text"].lower()
|
||||
|
||||
assert any(w in out_text for w in ("taxi", "cab", "car")), out_text
|
||||
|
||||
has_sg_or_logo_side = any(
|
||||
kw in out_text
|
||||
for kw in (
|
||||
"sg ",
|
||||
"sgl",
|
||||
" sgl",
|
||||
"logo",
|
||||
"software guidance",
|
||||
"labs",
|
||||
"laborator",
|
||||
"company",
|
||||
" text",
|
||||
)
|
||||
)
|
||||
assert has_sg_or_logo_side, out_text
|
||||
|
||||
def get_completion_request(self) -> ChatCompletionRequest:
|
||||
json_structure = {
|
||||
"model": self.model_path,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": self.image_urls[0]}},
|
||||
{"type": "image_url", "image_url": {"url": self.image_urls[1]}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe both the first image and the second image in detail separately.", # update prompt, ensure kimi-vl understands the images separately.
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
json_str = json.dumps(json_structure)
|
||||
return ChatCompletionRequest.model_validate_json(json_str)
|
||||
|
||||
def get_processor_output(self, req: Optional[ChatCompletionRequest] = None):
|
||||
if req is None:
|
||||
req = self.get_completion_request()
|
||||
conv = generate_chat_conv(req, template_name=self.chat_template)
|
||||
text = conv.get_prompt()
|
||||
|
||||
# Process inputs using processor
|
||||
inputs = self.processor(
|
||||
text=[text],
|
||||
images=self.main_image,
|
||||
return_tensors="pt",
|
||||
).to(self.device)
|
||||
|
||||
return inputs, text
|
||||
|
||||
async def test_accepts_image(self):
|
||||
req = self.get_completion_request()
|
||||
conv = generate_chat_conv(req, template_name=self.chat_template)
|
||||
text = conv.get_prompt()
|
||||
output = await self.engine.async_generate(
|
||||
prompt=text,
|
||||
image_data=self.main_image,
|
||||
sampling_params=dict(temperature=0.0, max_new_tokens=512),
|
||||
)
|
||||
self.verify_response(output)
|
||||
|
||||
async def test_accepts_precomputed_embeddings(self):
|
||||
req = self.get_completion_request()
|
||||
processor_output, _ = self.get_processor_output(req=req)
|
||||
|
||||
with torch.inference_mode():
|
||||
precomputed_embeddings = self.__class__.visual(processor_output)
|
||||
|
||||
output = await self.engine.async_generate(
|
||||
input_ids=processor_output["input_ids"][0].detach().cpu().tolist(),
|
||||
image_data=[
|
||||
self._precomputed_image_data(processor_output, precomputed_embeddings)
|
||||
],
|
||||
sampling_params=dict(temperature=0.0, max_new_tokens=512),
|
||||
)
|
||||
self.verify_response(output)
|
||||
|
||||
async def test_accepts_processor_output(self):
|
||||
req = self.get_completion_request()
|
||||
processor_output, prompt = self.get_processor_output(req=req)
|
||||
output = await self.engine.async_generate(
|
||||
input_ids=processor_output["input_ids"][0].detach().cpu().tolist(),
|
||||
image_data=[self._processor_output_image_data(processor_output)],
|
||||
sampling_params=dict(temperature=0.0, max_new_tokens=512),
|
||||
)
|
||||
self.verify_response(output)
|
||||
|
||||
def _precomputed_image_data(self, processor_output, precomputed_embeddings):
|
||||
"""This should not be overridden."""
|
||||
return dict(
|
||||
processor_output,
|
||||
format="precomputed_embedding",
|
||||
feature=precomputed_embeddings,
|
||||
)
|
||||
|
||||
def _processor_output_image_data(self, processor_output):
|
||||
"""Override in subclass to pass the correct set of arguments."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TestQwenVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
|
||||
model_path = "Qwen/Qwen2.5-VL-3B-Instruct"
|
||||
chat_template = "qwen2-vl"
|
||||
|
||||
@classmethod
|
||||
def _init_visual(cls):
|
||||
cls.visual_model = (
|
||||
Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||
cls.model_path, torch_dtype=torch.bfloat16
|
||||
)
|
||||
.eval()
|
||||
.visual.to(cls.device)
|
||||
)
|
||||
cls.visual = lambda processor_output: cls.visual_model(
|
||||
processor_output["pixel_values"], processor_output["image_grid_thw"]
|
||||
)
|
||||
|
||||
def _processor_output_image_data(self, processor_output):
|
||||
return dict(processor_output, format="processor_output")
|
||||
|
||||
|
||||
class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
|
||||
model_path = "google/gemma-3-4b-it"
|
||||
chat_template = "gemma-it"
|
||||
|
||||
@classmethod
|
||||
def _init_visual(cls):
|
||||
model = Gemma3ForConditionalGeneration.from_pretrained(
|
||||
cls.model_path, torch_dtype=torch.bfloat16
|
||||
)
|
||||
base_model = model.model
|
||||
|
||||
cls.vision_tower = base_model.vision_tower.eval().to(cls.device)
|
||||
|
||||
if hasattr(base_model, "multi_modal_projector"):
|
||||
cls.mm_projector = base_model.multi_modal_projector.eval().to(cls.device)
|
||||
else:
|
||||
cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
|
||||
|
||||
cls.visual = lambda processor_output: cls.mm_projector(
|
||||
cls.vision_tower(
|
||||
pixel_values=processor_output["pixel_values"]
|
||||
).last_hidden_state
|
||||
)
|
||||
|
||||
def _processor_output_image_data(self, processor_output):
|
||||
return dict(processor_output, format="processor_output")
|
||||
|
||||
|
||||
# Updated Kimi-VL test to use the new input format.
|
||||
class TestKimiVLImageUnderstandsImage(
|
||||
VLMInputTestBase, unittest.IsolatedAsyncioTestCase
|
||||
):
|
||||
model_path = "moonshotai/Kimi-VL-A3B-Instruct"
|
||||
chat_template = "kimi-vl"
|
||||
|
||||
@classmethod
|
||||
def _init_visual(cls):
|
||||
model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True)
|
||||
cls.vision_tower = model.vision_tower.eval().to(cls.device)
|
||||
cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
|
||||
|
||||
cls.visual = lambda tokenizer_output: cls.mm_projector(
|
||||
cls.vision_tower(
|
||||
pixel_values=tokenizer_output["pixel_values"],
|
||||
grid_hws=tokenizer_output["image_grid_hws"],
|
||||
)
|
||||
)
|
||||
|
||||
def _processor_output_image_data(self, processor_output):
|
||||
return dict(processor_output, format="processor_output")
|
||||
|
||||
|
||||
# not for CI: too large
|
||||
# class TestLlama4ImageUnderstandsImage(
|
||||
# VLMInputTestBase, unittest.IsolatedAsyncioTestCase
|
||||
# ):
|
||||
# # Allow overriding via env for local/offline runs.
|
||||
# model_path = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
|
||||
# chat_template = "llama-4"
|
||||
|
||||
# def setUp(self):
|
||||
# if torch.cuda.device_count() < 4:
|
||||
# self.skipTest("Skipping Llama-4 test: requires 4 GPUs for TP=4")
|
||||
# self.engine = Engine(
|
||||
# model_path=self.model_path,
|
||||
# trust_remote_code=True,
|
||||
# chat_template=self.chat_template,
|
||||
# enable_multimodal=True,
|
||||
# mem_fraction_static=0.8,
|
||||
# tp_size=4,
|
||||
# attention_backend="fa3",
|
||||
# context_length=65536,
|
||||
# )
|
||||
|
||||
# @classmethod
|
||||
# def _init_visual(cls):
|
||||
# model = AutoModel.from_pretrained(
|
||||
# cls.model_path,
|
||||
# trust_remote_code=True,
|
||||
# torch_dtype="auto",
|
||||
# force_download=True,
|
||||
# )
|
||||
# cls.vision_tower = model.vision_model.eval().to(cls.device)
|
||||
# cls.mm_projector = model.multi_modal_projector.eval().to(cls.device)
|
||||
|
||||
# cls.visual = lambda tokenizer_output: cls.mm_projector(
|
||||
# cls.vision_tower(
|
||||
# pixel_values=tokenizer_output["pixel_values"],
|
||||
# ).last_hidden_state.flatten(0, -2)
|
||||
# )
|
||||
|
||||
# def _processor_output_image_data(self, processor_output):
|
||||
# # Llama-4 vision expects processor_output format with pixel_values
|
||||
# return dict(processor_output, format="processor_output")
|
||||
|
||||
|
||||
# class TestLlavaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
|
||||
# model_path = "llava-hf/llava-1.5-7b-hf"
|
||||
# chat_template = "vicuna_v1.1"
|
||||
|
||||
# @classmethod
|
||||
# def _init_visual(cls):
|
||||
# from transformers import LlavaForConditionalGeneration
|
||||
|
||||
# model = LlavaForConditionalGeneration.from_pretrained(
|
||||
# cls.model_path,
|
||||
# torch_dtype=torch.float16,
|
||||
# low_cpu_mem_usage=True,
|
||||
# )
|
||||
# cls.vision_tower = model.vision_tower.eval().to(cls.device)
|
||||
# cls.multi_modal_projector = model.multi_modal_projector.eval().to(cls.device)
|
||||
# cls.config = model.config
|
||||
|
||||
# def visual_func(processor_output):
|
||||
# pixel_values = processor_output["pixel_values"].to(
|
||||
# cls.device, dtype=torch.float16
|
||||
# )
|
||||
|
||||
# vision_outputs = cls.vision_tower(pixel_values, output_hidden_states=True)
|
||||
# image_features = vision_outputs.hidden_states[-2]
|
||||
|
||||
# if cls.config.vision_feature_select_strategy == "default":
|
||||
# image_features = image_features[:, 1:]
|
||||
# elif cls.config.vision_feature_select_strategy == "full":
|
||||
# image_features = image_features
|
||||
|
||||
# image_features = cls.multi_modal_projector(image_features)
|
||||
# return image_features
|
||||
|
||||
# cls.visual = visual_func
|
||||
|
||||
# def _processor_output_image_data(self, processor_output):
|
||||
# return dict(processor_output, format="processor_output")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user