fix: duplicate resize images logic of qwen-vl series models (#12458)
Signed-off-by: yangsijia.614 <yangsijia.614@bytedance.com>
This commit is contained in:
@@ -1,16 +1,12 @@
|
|||||||
import asyncio
|
|
||||||
import re
|
import re
|
||||||
from typing import Dict, List, Union
|
from typing import Dict, List, Union
|
||||||
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from sglang.srt.models.dots_ocr import DotsOCRForCausalLM
|
from sglang.srt.models.dots_ocr import DotsOCRForCausalLM
|
||||||
from sglang.srt.models.dots_vlm import DotsVLMForCausalLM
|
from sglang.srt.models.dots_vlm import DotsVLMForCausalLM
|
||||||
from sglang.srt.multimodal.processors.base_processor import (
|
from sglang.srt.multimodal.processors.base_processor import (
|
||||||
BaseMultimodalProcessor,
|
BaseMultimodalProcessor,
|
||||||
MultimodalSpecialTokens,
|
MultimodalSpecialTokens,
|
||||||
)
|
)
|
||||||
from sglang.srt.multimodal.processors.qwen_vl import resize_image_async
|
|
||||||
|
|
||||||
|
|
||||||
class DotsVLMImageProcessor(BaseMultimodalProcessor):
|
class DotsVLMImageProcessor(BaseMultimodalProcessor):
|
||||||
@@ -70,18 +66,6 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor):
|
|||||||
multimodal_tokens=self.mm_tokens,
|
multimodal_tokens=self.mm_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Qwen-specific: resize images if they are raw Image objects
|
|
||||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
|
||||||
resize_tasks = [
|
|
||||||
resize_image_async(
|
|
||||||
image,
|
|
||||||
min_pixels=self.MIN_PIXELS,
|
|
||||||
max_pixels=self.MAX_PIXELS,
|
|
||||||
size_factor=self.IMAGE_FACTOR,
|
|
||||||
)
|
|
||||||
for image in base_output.images
|
|
||||||
]
|
|
||||||
base_output.images = await asyncio.gather(*resize_tasks)
|
|
||||||
combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
|
combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
|
||||||
base_output, self.mm_tokens
|
base_output, self.mm_tokens
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,15 +1,9 @@
|
|||||||
# Copy from qwen_vl.py, adapted for points-v15-chat
|
# Copy from qwen_vl.py, adapted for points-v15-chat
|
||||||
|
|
||||||
import asyncio
|
|
||||||
from typing import List, Union
|
from typing import List, Union
|
||||||
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from sglang.srt.models.points_v15_chat import POINTSV15ChatModel
|
from sglang.srt.models.points_v15_chat import POINTSV15ChatModel
|
||||||
from sglang.srt.multimodal.processors.qwen_vl import (
|
from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
|
||||||
QwenVLImageProcessor,
|
|
||||||
resize_image_async,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class POINTSV15ChatProcessor(QwenVLImageProcessor):
|
class POINTSV15ChatProcessor(QwenVLImageProcessor):
|
||||||
@@ -37,10 +31,6 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor):
|
|||||||
multimodal_tokens=self.mm_tokens,
|
multimodal_tokens=self.mm_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
|
||||||
resize_tasks = [resize_image_async(image) for image in base_output.images]
|
|
||||||
base_output.images = await asyncio.gather(*resize_tasks)
|
|
||||||
|
|
||||||
mm_items, input_ids, _ = self.process_and_combine_mm_data(
|
mm_items, input_ids, _ = self.process_and_combine_mm_data(
|
||||||
base_output, self.mm_tokens
|
base_output, self.mm_tokens
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import asyncio
|
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@@ -79,26 +78,6 @@ def smart_resize(
|
|||||||
return h_bar, w_bar
|
return h_bar, w_bar
|
||||||
|
|
||||||
|
|
||||||
def resize_image(
|
|
||||||
image,
|
|
||||||
min_pixels: int = MIN_PIXELS,
|
|
||||||
max_pixels: int = MAX_PIXELS,
|
|
||||||
size_factor: int = IMAGE_FACTOR,
|
|
||||||
) -> Image.Image:
|
|
||||||
width, height = image.size
|
|
||||||
min_pixels = min_pixels
|
|
||||||
max_pixels = max_pixels
|
|
||||||
resized_height, resized_width = smart_resize(
|
|
||||||
height,
|
|
||||||
width,
|
|
||||||
factor=size_factor,
|
|
||||||
min_pixels=min_pixels,
|
|
||||||
max_pixels=max_pixels,
|
|
||||||
)
|
|
||||||
image = image.resize((resized_width, resized_height), resample=RESIZE_RESAMPLE)
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def round_by_factor(number: int, factor: int) -> int:
|
def round_by_factor(number: int, factor: int) -> int:
|
||||||
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
||||||
return round(number / factor) * factor
|
return round(number / factor) * factor
|
||||||
@@ -114,15 +93,6 @@ def floor_by_factor(number: int, factor: int) -> int:
|
|||||||
return math.floor(number / factor) * factor
|
return math.floor(number / factor) * factor
|
||||||
|
|
||||||
|
|
||||||
async def resize_image_async(
|
|
||||||
image,
|
|
||||||
min_pixels: int = MIN_PIXELS,
|
|
||||||
max_pixels: int = MAX_PIXELS,
|
|
||||||
size_factor: int = IMAGE_FACTOR,
|
|
||||||
):
|
|
||||||
return resize_image(image, min_pixels, max_pixels, size_factor)
|
|
||||||
|
|
||||||
|
|
||||||
def smart_nframes(
|
def smart_nframes(
|
||||||
ele: dict,
|
ele: dict,
|
||||||
total_frames: int,
|
total_frames: int,
|
||||||
@@ -266,11 +236,6 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
|||||||
self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
|
self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
|
||||||
self.audio_token_id = getattr(hf_config, "audio_token_id", None)
|
self.audio_token_id = getattr(hf_config, "audio_token_id", None)
|
||||||
|
|
||||||
self.NUM_TOKEN_PER_FRAME = 770
|
|
||||||
self.IMAGE_FACTOR = 28
|
|
||||||
self.MIN_PIXELS = 4 * 28 * 28
|
|
||||||
self.MAX_PIXELS = 16384 * 28 * 28
|
|
||||||
self.MAX_RATIO = 200
|
|
||||||
self.mm_tokens = MultimodalSpecialTokens(
|
self.mm_tokens = MultimodalSpecialTokens(
|
||||||
image_token="<|vision_start|><|image_pad|><|vision_end|>",
|
image_token="<|vision_start|><|image_pad|><|vision_end|>",
|
||||||
image_token_id=hf_config.image_token_id,
|
image_token_id=hf_config.image_token_id,
|
||||||
@@ -301,11 +266,6 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
|||||||
load_time = time.perf_counter()
|
load_time = time.perf_counter()
|
||||||
rid = getattr(request_obj, "rid", "anonymous_rid")
|
rid = getattr(request_obj, "rid", "anonymous_rid")
|
||||||
|
|
||||||
# Qwen-specific: resize images if they are raw Image objects
|
|
||||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
|
||||||
resize_tasks = [resize_image_async(image) for image in base_output.images]
|
|
||||||
base_output.images = await asyncio.gather(*resize_tasks)
|
|
||||||
|
|
||||||
video_metadata = None
|
video_metadata = None
|
||||||
if base_output.videos:
|
if base_output.videos:
|
||||||
videos_processed = [
|
videos_processed = [
|
||||||
|
|||||||
Reference in New Issue
Block a user