minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[VLM] feat: size the multimodal preprocessing pool by where preprocessing runs (#35349)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-08-27 16:32:58 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent c608f9bf75
commit c2c3320cf0
9 changed files with 450 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/multimodal/processors/base_processor.py
+66 -11
View File
@@ -191,6 +191,17 @@ class MultimodalSpecialTokens:
return self.combined_regex
def _tokenizer_of(processor):
"""The tokenizer reached from an HF processor.
Some processors (e.g. InternVL) are handed a tokenizer directly as their
``_processor`` rather than one that wraps a tokenizer. Every path that
resolves a tokenizer -- construction and per-worker processor clones alike --
goes through here, so a clone cannot resolve differently from the original.
"""
return processor.tokenizer if hasattr(processor, "tokenizer") else processor
class BaseMultimodalProcessor(ABC):
models = []
gpu_image_decode = True # Enable GPU decoding by default
@@ -199,12 +210,20 @@ class BaseMultimodalProcessor(ABC):
# Set by processors that already build input_ids from the request's own
# tokens, so the retokenize-avoidance rebuild below has nothing to add.
preserve_processor_input_ids = False
auto_mm_processor_worker_num = 1
# None lets the worker count follow where preprocessing actually runs; a
# model that measured its own optimum assigns a number instead. See
# `_resolve_auto_mm_processor_worker_num`.
auto_mm_processor_worker_num = None
auto_mm_io_worker_num = 4
# Models opt in by assigning a non-zero default. A user-provided server
# argument overrides this value; zero disables storage and cache-key work.
auto_mm_preprocess_cache_size_mb = 0
supports_mm_processor_concurrency = False
# Processors opt out only when their preprocessing is not thread-safe. The
# worker pool gives each thread its own `copy.deepcopy` of the HF processor
# and injects it, and the single function it runs --
# `process_and_combine_mm_data` -- resolves that clone instead of
# `self._processor`, so isolation does not depend on the subclass.
supports_mm_processor_concurrency = True
def __init__(
self, hf_config, server_args, _processor, transport_mode, *args, **kwargs
@@ -272,12 +291,7 @@ class BaseMultimodalProcessor(ABC):
"trusted" if self.trust_mm_content_hashes else "verified",
)
# Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer
# directly as _processor rather than a processor that wraps a tokenizer.
if hasattr(self._processor, "tokenizer"):
self._tokenizer = self._processor.tokenizer
else:
self._tokenizer = self._processor
self._tokenizer = _tokenizer_of(self._processor)
# Same guard as in serving_chat.py against double BOS.
try:
@@ -314,7 +328,8 @@ class BaseMultimodalProcessor(ABC):
self.mm_processor_worker_num = (
1
if skip_mm_pool
else requested_mm_processor_worker_num or self.auto_mm_processor_worker_num
else requested_mm_processor_worker_num
or self._resolve_auto_mm_processor_worker_num()
)
if (
self.mm_processor_worker_num > 1
@@ -329,8 +344,11 @@ class BaseMultimodalProcessor(ABC):
self.mm_processor_executor = None
if self.mm_processor_worker_num > 1:
try:
# A callable, not the object: subclasses finish customizing
# `_processor` after this returns, and the workers must clone it
# as the subclass left it.
self.mm_processor_executor = MultimodalProcessorExecutor(
self._processor, self.mm_processor_worker_num
lambda: self._processor, self.mm_processor_worker_num
)
except Exception:
logger.warning(
@@ -594,7 +612,44 @@ class BaseMultimodalProcessor(ABC):
def _resolve_processor(self, processor=None):
if processor is None:
return self._processor, self._tokenizer
return processor, processor.tokenizer
return processor, _tokenizer_of(processor)
def _preprocessing_competes_with_the_scheduler(self) -> bool:
"""Whether image preprocessing submits its work to the serving GPU.
The fast image processor runs inside the tokenizer process but on
``cuda:{base_gpu_id}`` -- the device the scheduler serves from. A second
preprocessing worker there is one more competitor for that device rather
than added parallelism.
"""
if _is_cpu or self.server_args.rl_on_policy_target is not None:
return False
if self.disable_fast_image_processor:
return False
image_processor = getattr(self._processor, "image_processor", None)
return isinstance(image_processor, BaseImageProcessor)
def _resolve_auto_mm_processor_worker_num(self) -> int:
"""The worker count to use when the user did not ask for one.
Two workers overlap preprocessing that runs on the CPU, where the second
thread is real parallelism: measured on Qwen2.5-VL with full-page images
at 32-way concurrency, 4.46 -> 6.08 req/s on H200 and 7.07 -> 8.76 on
GB300.
The GPU path is capped at one worker even when a model declares more.
A declaration records what its author measured on one platform and one
image shape; contending for the device the scheduler is serving from is a
property of the path itself, and it does not go away because a subclass
asked for concurrency. Qwen-VL declares two and is the model that
measures 9.30 -> 4.02 req/s on GB300 full-page images, so honouring the
declaration here would exempt exactly the case that regresses.
`--mm-processor-worker-num` still overrides this.
"""
if self._preprocessing_competes_with_the_scheduler():
return 1
declared = self.auto_mm_processor_worker_num
return 2 if declared is None else declared
def _fast_image_processor_device(self, processor) -> Optional[str]:
"""The device for the fast image processor, or None to leave it unset.
python/sglang/srt/multimodal/processors/ernie45_vl.py
+10 -2
View File
@@ -282,7 +282,13 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
return pixel_values
def process_mm_data(
self, input_text, images=None, videos=None, audios=None, **kwargs
self,
input_text,
images=None,
videos=None,
audios=None,
processor=None,
**kwargs,
) -> dict:
"""
process multimodal data with transformers AutoProcessor
@@ -296,7 +302,9 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
if self.video_config:
kwargs.setdefault("videos_kwargs", {}).update(self.video_config)
processor = self._processor
# Take the worker pool's per-thread clone when it hands one over; falling
# back to self._processor would put every worker on one shared object.
processor, _ = self._resolve_processor(processor)
if (
hasattr(processor, "image_processor")
and isinstance(processor.image_processor, BaseImageProcessor)
python/sglang/srt/multimodal/processors/executor.py
+11 -8
View File
@@ -15,9 +15,14 @@ class _WorkerState(threading.local):
class MultimodalProcessorExecutor:
"""Run processor calls on isolated, thread-local processor clones."""
def __init__(self, processor: Any, max_workers: int):
self._processor = processor
self._processor_clones = [copy.deepcopy(processor) for _ in range(max_workers)]
def __init__(self, resolve_processor: Callable[[], Any], max_workers: int):
# Resolved per clone rather than captured here: a subclass keeps
# customizing `_processor` after `super().__init__()` has already built
# this pool, and a clone taken now would miss every one of those edits.
self._resolve_processor = resolve_processor
# Probe once, so a processor that cannot be cloned still falls back to
# synchronous processing at startup instead of failing inside a worker.
copy.deepcopy(resolve_processor())
self._executor = concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers,
thread_name_prefix="sglang-mm-processor",
@@ -39,12 +44,10 @@ class MultimodalProcessorExecutor:
) -> T:
processor = self._worker_state.processor
if processor is None:
# One clone at a time: cloning reads the shared processor, which the
# worker path also reads for the token-count helpers.
with self._clone_lock:
processor = (
self._processor_clones.pop()
if self._processor_clones
else copy.deepcopy(self._processor)
)
processor = copy.deepcopy(self._resolve_processor())
self._worker_state.processor = processor
return function(*args, processor=processor, **kwargs)
python/sglang/srt/multimodal/processors/midashenglm.py
+10 -2
View File
@@ -48,7 +48,13 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
self.FEATURE_NAMES.append("input_values")
def process_mm_data(
self, input_text, images=None, videos=None, audios=None, **kwargs
self,
input_text,
images=None,
videos=None,
audios=None,
processor=None,
**kwargs,
):
"""Override to use correct audio parameter name for MiDashengLM processor."""
if images:
@@ -62,7 +68,9 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
if self.audio_config:
kwargs["audio_kwargs"].update(self.audio_config)
processor = self._processor
# Take the worker pool's per-thread clone when it hands one over; falling
# back to self._processor would put every worker on one shared object.
processor, _ = self._resolve_processor(processor)
result = processor.__call__(
text=[input_text],
padding=True,
python/sglang/srt/multimodal/processors/sarashina2_vision.py
+40 -25
View File
@@ -1,3 +1,4 @@
import types
from typing import List, Union
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
@@ -7,6 +8,43 @@ from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
# Sarashina2Vision's remote-code `_preprocess` takes a narrow kwarg set, while
# transformers' `preprocess` forwards its full one, so the extras have to be
# dropped.
_PREPROCESS_PARAMS = frozenset(
{
"do_resize",
"resample",
"do_rescale",
"rescale_factor",
"do_normalize",
"image_mean",
"image_std",
"do_convert_rgb",
"data_format",
"input_data_format",
}
)
def _install_preprocess_kwarg_filter(image_processor) -> None:
"""Drop the kwargs Sarashina2Vision's `_preprocess` cannot accept.
Bound with `types.MethodType` rather than closing over `image_processor`, so
that a preprocessing worker's `copy.deepcopy` rebinds `__self__` to its own
clone instead of routing every thread back into this one.
"""
unfiltered_preprocess = type(image_processor)._preprocess
def _preprocess(self, *args, **kwargs):
return unfiltered_preprocess(
self,
*args,
**{k: v for k, v in kwargs.items() if k in _PREPROCESS_PARAMS},
)
image_processor._preprocess = types.MethodType(_preprocess, image_processor)
class Sarashina2VisionProcessor(BaseMultimodalProcessor):
models = [Sarashina2VisionForCausalLM]
@@ -25,33 +63,10 @@ class Sarashina2VisionProcessor(BaseMultimodalProcessor):
image_token_id=self.IM_TOKEN_ID,
).build(_processor)
# Patch the processor's image processor to handle parameter compatibility
if hasattr(_processor, "image_processor") and hasattr(
_processor.image_processor, "_preprocess"
type(_processor.image_processor), "_preprocess"
):
original_preprocess = _processor.image_processor._preprocess
def patched_preprocess(*args, **kwargs):
# Filter kwargs to only include parameters that the custom _preprocess method accepts
# Based on Sarashina2VisionImageProcessor._preprocess signature
allowed_params = {
"do_resize",
"resample",
"do_rescale",
"rescale_factor",
"do_normalize",
"image_mean",
"image_std",
"do_convert_rgb",
"data_format",
"input_data_format",
}
filtered_kwargs = {
k: v for k, v in kwargs.items() if k in allowed_params
}
return original_preprocess(*args, **filtered_kwargs)
_processor.image_processor._preprocess = patched_preprocess
_install_preprocess_kwarg_filter(_processor.image_processor)
async def process_mm_data_async(
self,