minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[VLM] replace deprecated image processor use_fast (#34175)

Browse Source
This commit is contained in:
Mick
2026-08-12 00:14:07 +08:00
committed by GitHub
parent b20c375c10
commit 8267d76c2c
21 changed files with 281 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs/advanced_features/server_arguments.mdx
+7 -1
View File
@@ -2693,9 +2693,15 @@ Please consult the documentation below and [server_args.py](https://github.com/s
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--image-processor-backend`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Image processor backend. `auto` lets Transformers select the best available backend.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`auto`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`auto`, `torchvision`, `pil`</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--disable-fast-image-processor`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Adopt base image processor instead of fast image processor.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Deprecated. Use `--image-processor-backend=pil` instead.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
docs/docs/advanced_features/vlm_query.mdx
+4 -4
View File
@@ -89,7 +89,7 @@ Using a HuggingFace processor to preprocess text and images, and passing the `pr
```python Example
from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained(model_path, use_fast=True)
processor = AutoProcessor.from_pretrained(model_path)
processor_output = processor(
images=[image], text=conv.get_prompt(), return_tensors="pt"
)
@@ -110,7 +110,7 @@ You can pre-calculate image features to avoid repeated visual encoding processes
from transformers import AutoProcessor
from transformers import Qwen2_5_VLForConditionalGeneration
processor = AutoProcessor.from_pretrained(model_path, use_fast=True)
processor = AutoProcessor.from_pretrained(model_path)
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path).eval()
vision = model.model.visual.cuda()
```
@@ -192,7 +192,7 @@ Using HuggingFace processor to preprocess data can reduce computational overhead
```python Example
from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained(model_path, use_fast=True)
processor = AutoProcessor.from_pretrained(model_path)
processor_output = processor(
images=[image], text=conv.get_prompt(), return_tensors="pt"
)
@@ -211,7 +211,7 @@ print(out)
from transformers import AutoProcessor
from transformers import Llama4ForConditionalGeneration
processor = AutoProcessor.from_pretrained(model_path, use_fast=True)
processor = AutoProcessor.from_pretrained(model_path)
model = Llama4ForConditionalGeneration.from_pretrained(
model_path, torch_dtype="auto"
).eval()
docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx
+6
View File
@@ -2336,6 +2336,12 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>bool flag<br/> (set to enable)</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>A2, A3</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--image-processor-backend`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`auto`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`auto`, `torchvision`, `pil`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>A2, A3</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--disable-fast-image-processor`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`False`</td>
docs/docs/sglang-diffusion/models_with_ar.mdx
+1 -1
View File
@@ -109,7 +109,7 @@ export HCCL_HOST_SOCKET_PORT_RANGE="23000-23199"
export HCCL_NPU_SOCKET_PORT_RANGE="23200-23399"
sglang serve --model-path /path/to/zai-org/GLM-Image/vision_language_encoder/ \
--tokenizer-path /path/to/zai-org/GLM-Image/processor/ --enable-multimodal \
--cuda-graph-bs 1 --device npu --attention-backend ascend --disable-fast-image-processor \
--cuda-graph-bs 1 --device npu --attention-backend ascend --image-processor-backend pil \
--tp-size 4 --port ${PORT} --mem-fraction-static 0.4
```
Second terminal with diffusion server:
python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py
+3 -1
View File
@@ -480,7 +480,9 @@ class ImageProcessorLoader(ComponentLoader):
def load_customized(
self, component_model_path: str, server_args: ServerArgs, component_name: str
) -> Any:
return AutoImageProcessor.from_pretrained(component_model_path, use_fast=True)
return AutoImageProcessor.from_pretrained(
component_model_path, backend="torchvision"
)
class AutoProcessorLoader(ComponentLoader):
python/sglang/multimodal_gen/test/single_test_file/test_ar_models.py
+2 -1
View File
@@ -73,7 +73,8 @@ class ARCluster(DisaggCluster):
"--enable-multimodal",
"--cuda-graph-bs",
"1",
"--disable-fast-image-processor",
"--image-processor-backend",
"pil",
"--tp-size",
str(len(gpus)),
"--port",
python/sglang/multimodal_gen/test/test_utils.py
+1 -2
View File
@@ -914,12 +914,11 @@ def get_clip_model() -> tuple[Any, Any]:
if "RobertaProcessing" not in str(e):
raise
logger.warning(
"Fast CLIP processor failed (%s), retrying with use_fast=False", e
"CLIP processor failed (%s), retrying with compatibility shim", e
)
processor = _load_clip_processor_with_roberta_processing_compat(
CLIPProcessor,
CLIP_MODEL_NAME,
use_fast=False,
)
model = CLIPModel.from_pretrained(CLIP_MODEL_NAME)
python/sglang/srt/disaggregation/encode_receiver.py
+12 -27
View File
@@ -36,7 +36,10 @@ from sglang.srt.managers.schedule_batch import Modality, Req
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import ImageData
from sglang.srt.utils.common import safe_pickle_loads
from sglang.srt.utils.hf_transformers_utils import get_processor
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
resolve_image_processor_backend,
)
from sglang.srt.utils.network import (
NetworkAddress,
get_local_ip_auto,
@@ -1674,32 +1677,14 @@ class MMReceiverBase(ABC):
if getattr(server_args, "tokenizer_backend", None) is not None:
extra_kwargs["tokenizer_backend"] = server_args.tokenizer_backend
_processor = None
try:
_processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
**extra_kwargs,
)
except ValueError as e:
error_message = str(e)
if "does not have a slow version" in error_message:
logger.info(
f"Processor {server_args.tokenizer_path} does not have a slow version. Automatically use fast version"
)
_processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=True,
**extra_kwargs,
)
else:
raise e
_processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
image_processor_backend=resolve_image_processor_backend(server_args),
**extra_kwargs,
)
enable_adaptive_dispatch_to_encoder = (
server_args.enable_adaptive_dispatch_to_encoder
python/sglang/srt/disaggregation/encode_server.py
+10 -4
View File
@@ -92,6 +92,7 @@ from sglang.srt.utils import (
set_prometheus_multiproc_dir,
)
from sglang.srt.utils.common import configure_logger, maybe_reindex_device_id
from sglang.srt.utils.hf_transformers_utils import resolve_image_processor_backend
from sglang.srt.utils.network import (
NetworkAddress,
config_socket,
@@ -341,7 +342,8 @@ class MMEncoder:
torch.get_device_module(self.device).set_device(self.gpu_id)
self.use_image_processor_gpu = (
use_image_processor_gpu and not server_args.disable_fast_image_processor
use_image_processor_gpu
and resolve_image_processor_backend(server_args) != "pil"
)
self._build_vision_config(server_args.mm_process_config)
self.model_audio_sr = self._resolve_audio_sr()
@@ -606,12 +608,18 @@ class MMEncoder:
"""
from transformers import AutoImageProcessor, AutoVideoProcessor
image_processor_backend = resolve_image_processor_backend(server_args)
image_processor_kwargs = (
{}
if image_processor_backend == "auto"
else {"backend": image_processor_backend}
)
try:
self.image_processor = AutoImageProcessor.from_pretrained(
server_args.tokenizer_path or server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
**image_processor_kwargs,
)
except Exception as e:
logger.warning(f"Failed to load image processor: {e}")
@@ -622,7 +630,6 @@ class MMEncoder:
server_args.tokenizer_path or server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
)
except Exception as e:
logger.warning(f"Failed to load video processor: {e}")
@@ -634,7 +641,6 @@ class MMEncoder:
server_args.tokenizer_path or server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
)
if not hasattr(_audio_proc, "feature_extractor"):
logger.warning(
python/sglang/srt/managers/scheduler.py
+2 -1
View File
@@ -315,6 +315,7 @@ from sglang.srt.utils.hf_transformers_utils import (
get_processor,
get_tokenizer,
get_tokenizer_from_processor,
resolve_image_processor_backend,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.utils.numa_utils import get_numa_node_if_available, numa_bind_to_node
@@ -812,7 +813,7 @@ class Scheduler(
tokenizer_mode=get_serving().tokenizer_mode,
trust_remote_code=get_model().trust_remote_code,
revision=get_model().revision,
use_fast=not get_mm().disable_fast_image_processor,
image_processor_backend=resolve_image_processor_backend(get_mm()),
tokenizer_backend=get_serving().tokenizer_backend,
model_name=get_model().model_path,
)
python/sglang/srt/managers/tokenizer_manager.py
+10 -28
View File
@@ -145,6 +145,7 @@ from sglang.srt.utils.hf_transformers_utils import (
get_processor,
get_tokenizer,
get_tokenizer_from_processor,
resolve_image_processor_backend,
)
from sglang.srt.utils.network import get_zmq_socket
from sglang.srt.utils.request_logger import RequestLogger
@@ -3556,34 +3557,15 @@ async def print_exception_wrapper(func):
def get_processor_wrapper(server_args):
try:
processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
tokenizer_backend=server_args.tokenizer_backend,
model_name=server_args.model_path,
)
except ValueError as e:
error_message = str(e)
if "does not have a slow version" in error_message:
logger.info(
f"Processor {server_args.tokenizer_path} does not have a slow version. Automatically use fast version"
)
processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=True,
tokenizer_backend=server_args.tokenizer_backend,
model_name=server_args.model_path,
)
else:
raise e
return processor
return get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
image_processor_backend=resolve_image_processor_backend(server_args),
tokenizer_backend=server_args.tokenizer_backend,
model_name=server_args.model_path,
)
def determine_tensor_transport_mode(server_args: ServerArgs) -> TensorTransportMode:
python/sglang/srt/multimodal/processors/base_processor.py
+6 -1
View File
@@ -206,7 +206,12 @@ class BaseMultimodalProcessor(ABC):
self.use_ipc_pool_handle_cache = (
self.use_cuda_ipc and envs.SGLANG_USE_IPC_POOL_HANDLE_CACHE.get()
)
self.disable_fast_image_processor = server_args.disable_fast_image_processor
self.image_processor_backend = getattr(
server_args, "image_processor_backend", "auto"
)
if getattr(server_args, "disable_fast_image_processor", False):
self.image_processor_backend = "pil"
self.disable_fast_image_processor = self.image_processor_backend == "pil"
self.skip_tokenizer_init = server_args.skip_tokenizer_init
mm_process_config = self.server_args.mm_process_config
python/sglang/srt/server_args.py
+21 -1
View File
@@ -2766,6 +2766,12 @@ class ServerArgs:
"Enable global multimodal embedding cache to skip redundant ViT inference.",
NS("mm"),
] = False
image_processor_backend: A[
Literal["auto", "torchvision", "pil"],
"Image processor backend. 'auto' lets Transformers select the best "
"available backend.",
NS("mm"),
] = "auto"
mm_global_cache_backend: A[
str,
Arg(
@@ -2776,7 +2782,9 @@ class ServerArgs:
NS("mm"),
] = "mooncake"
disable_fast_image_processor: A[
bool, "Adopt base image processor instead of fast image processor.", NS("mm")
bool,
"Deprecated. Use --image-processor-backend=pil instead.",
NS("mm"),
] = False
mm_feature_transport: A[
Optional[Literal["cpu", "cuda_ipc", "cuda_vmm"]],
@@ -3990,6 +3998,18 @@ class ServerArgs:
)
def _handle_deprecated_args(self):
if self.disable_fast_image_processor:
if self.image_processor_backend not in {"auto", "pil"}:
raise ValueError(
"--disable-fast-image-processor conflicts with "
f"--image-processor-backend={self.image_processor_backend}."
)
logger.warning(
"--disable-fast-image-processor is deprecated; use "
"--image-processor-backend=pil instead."
)
self.image_processor_backend = "pil"
# Handle deprecated tool call parsers
deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
if self.tool_call_parser in deprecated_tool_call_parsers:
python/sglang/srt/utils/hf_transformers/__init__.py
+2 -1
View File
@@ -37,7 +37,7 @@ from .common import (
get_tokenizer_from_processor,
)
from .config import get_config
from .processor import get_processor
from .processor import get_processor, resolve_image_processor_backend
from .tokenizer import (
_fix_added_tokens_encoding,
_fix_v5_add_bos_eos_token,
@@ -57,6 +57,7 @@ __all__ = [
"get_generation_config",
"get_hf_text_config",
"get_processor",
"resolve_image_processor_backend",
"get_rope_config",
"get_sparse_attention_config",
"get_tokenizer",
python/sglang/srt/utils/hf_transformers/processor.py
+84 -20
View File
@@ -18,6 +18,7 @@ from pathlib import Path
from typing import Optional
from transformers import (
AutoImageProcessor,
AutoProcessor,
AutoTokenizer,
PreTrainedTokenizerBase,
@@ -49,6 +50,72 @@ from .tokenizer import (
_fix_special_tokens_pattern,
)
_IMAGE_PROCESSOR_BACKENDS = {"auto", "torchvision", "pil"}
def resolve_image_processor_backend(server_args) -> str:
"""Resolve the new backend option while honoring the legacy disable flag."""
if getattr(server_args, "disable_fast_image_processor", False):
return "pil"
return getattr(server_args, "image_processor_backend", "auto")
def _normalize_image_processor_backend(
image_processor_backend: Optional[str], use_fast: Optional[bool]
) -> str:
backend = image_processor_backend or "auto"
if backend not in _IMAGE_PROCESSOR_BACKENDS:
raise ValueError(
f"Unsupported image processor backend: {backend}. "
f"Expected one of {sorted(_IMAGE_PROCESSOR_BACKENDS)}."
)
if use_fast is not None:
legacy_backend = "torchvision" if use_fast else "pil"
if backend not in {"auto", legacy_backend}:
raise ValueError(
f"use_fast={use_fast} conflicts with "
f"image_processor_backend={backend!r}."
)
backend = legacy_backend
return backend
def _apply_image_processor_backend(
processor,
tokenizer_name,
args,
trust_remote_code,
revision,
backend,
kwargs,
):
"""Apply an explicit backend only to the image sub-processor.
ProcessorMixin forwards generic kwargs to every sub-processor. Passing
``backend`` through AutoProcessor therefore also reaches tokenizers and
video processors, where it has different semantics or may be read-only.
"""
if backend == "auto" or not hasattr(processor, "image_processor"):
return processor
image_processor = processor.image_processor
if getattr(image_processor, "backend", None) == backend:
return processor
image_processor_kwargs = dict(kwargs)
image_processor_kwargs.pop("backend", None)
image_processor_kwargs.pop("use_fast", None)
processor.image_processor = AutoImageProcessor.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
backend=backend,
**image_processor_kwargs,
)
return processor
def _build_processor_manually(
model_path, config, trust_remote_code, revision, **kwargs
@@ -141,7 +208,8 @@ def get_processor(
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
tokenizer_revision: Optional[str] = None,
use_fast: Optional[bool] = True,
use_fast: Optional[bool] = None,
image_processor_backend: Optional[str] = None,
tokenizer_backend: str = "huggingface",
model_name: Optional[str] = None,
**kwargs,
@@ -152,6 +220,9 @@ def get_processor(
_ensure_fastokens_patched()
revision = kwargs.pop("revision", tokenizer_revision)
image_processor_backend = _normalize_image_processor_backend(
image_processor_backend, use_fast
)
tokenizer_name = resolve_runai_obj_uri(tokenizer_name)
if model_name is not None:
model_name = resolve_runai_obj_uri(model_name)
@@ -186,7 +257,6 @@ def get_processor(
# Checkpoints with language_model_only=True are text-only despite their
# multimodal-family config; route to tokenizer instead of the mm processor.
if getattr(config, "language_model_only", False):
kwargs.pop("use_fast", None)
return AutoTokenizer.from_pretrained(
tokenizer_name,
*args,
@@ -199,8 +269,6 @@ def get_processor(
if "size" not in kwargs:
kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}
if config.model_type not in {"llava", "clip"}:
kwargs["use_fast"] = use_fast
try:
if "InternVL3_5" in tokenizer_name:
processor = AutoTokenizer.from_pretrained(
@@ -230,20 +298,7 @@ def get_processor(
except ValueError as e:
error_message = str(e)
if "does not have a slow version" in error_message:
logger.info(
"Processor %s does not have a slow version. Automatically use fast version",
tokenizer_name,
)
kwargs["use_fast"] = True
processor = AutoProcessor.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
elif "Unrecognized feature extractor" in error_message:
if "Unrecognized feature extractor" in error_message:
logger.info(
"AutoProcessor failed on feature extractor for %s, "
"constructing processor manually",
@@ -261,10 +316,9 @@ def get_processor(
):
logger.info(
"AutoProcessor for %s rejected standard kwargs, "
"retrying without trust_remote_code/use_fast",
"retrying without trust_remote_code",
tokenizer_name,
)
kwargs.pop("use_fast", None)
kwargs.pop("_from_auto", None)
processor = AutoProcessor.from_pretrained(
tokenizer_name,
@@ -274,6 +328,16 @@ def get_processor(
)
else:
raise
processor = _apply_image_processor_backend(
processor,
tokenizer_name,
args,
trust_remote_code,
revision,
image_processor_backend,
kwargs,
)
if (
isinstance(processor, PreTrainedTokenizerBase)
and getattr(config, "model_type", None) == "pixtral"
test/registered/unit/server_args/test_server_args.py
+35
View File
@@ -142,6 +142,41 @@ class TestMmEncoderDataParallelLogging(CustomTestCase):
self.assertIn("high-resolution or multi-image", logs.output[0])
class TestImageProcessorBackend(CustomTestCase):
def test_new_backend_does_not_set_legacy_flag(self):
server_args = ServerArgs(model_path="dummy", image_processor_backend="pil")
server_args._handle_deprecated_args()
self.assertEqual(server_args.image_processor_backend, "pil")
self.assertFalse(server_args.disable_fast_image_processor)
def test_legacy_flag_maps_to_pil_with_one_warning(self):
server_args = ServerArgs(model_path="dummy", disable_fast_image_processor=True)
with self.assertLogs(server_args_module.logger, level="WARNING") as logs:
server_args._handle_deprecated_args()
self.assertEqual(server_args.image_processor_backend, "pil")
self.assertTrue(server_args.disable_fast_image_processor)
self.assertEqual(
sum(
"--disable-fast-image-processor is deprecated" in x for x in logs.output
),
1,
)
def test_legacy_flag_rejects_torchvision_backend(self):
server_args = ServerArgs(
model_path="dummy",
image_processor_backend="torchvision",
disable_fast_image_processor=True,
)
with self.assertRaisesRegex(ValueError, "conflicts.*torchvision"):
server_args._handle_deprecated_args()
class TestMultimodalFeatureTransport(CustomTestCase):
@staticmethod
def _set_model_type(server_args, *, is_multimodal):
test/registered/unit/test_server_args_migration.py
+6
View File
@@ -87,6 +87,12 @@ class TestServerArgsAnnotatedCli(CustomTestCase):
self.assertEqual(sa.deepep_mode, "low_latency")
self.assertEqual(sa.elastic_ep_backend, "none")
def test_image_processor_backend_choices(self):
for backend in ("auto", "torchvision", "pil"):
with self.subTest(backend=backend):
sa = self._parse(["--image-processor-backend", backend])
self.assertEqual(sa.image_processor_backend, backend)
def test_deprecated_flags_still_work(self):
"""Deprecated flags set the correct dest field."""
sa = self._parse(["--stream-output"])
test/registered/unit/utils/test_hf_transformers.py
+66
View File
@@ -39,6 +39,72 @@ register_cpu_ci(est_time=6, suite="base-a-test-cpu")
class TestGetProcessor(unittest.TestCase):
def test_does_not_forward_backend_to_auto_processor(self):
config = SimpleNamespace(model_type="test_vlm", auto_map={})
loaded_processor = MagicMock()
loaded_processor.image_processor.backend = "torchvision"
loaded_processor.tokenizer.chat_template = "template"
auto_config = MagicMock()
auto_config.from_pretrained.return_value = config
auto_processor = MagicMock()
auto_processor.from_pretrained.return_value = loaded_processor
auto_image_processor = MagicMock()
with patch.multiple(
processor_utils,
AutoConfig=auto_config,
AutoProcessor=auto_processor,
AutoImageProcessor=auto_image_processor,
):
processor_utils.get_processor(
"test-model", image_processor_backend="torchvision"
)
call_kwargs = auto_processor.from_pretrained.call_args.kwargs
self.assertNotIn("backend", call_kwargs)
self.assertNotIn("use_fast", call_kwargs)
auto_image_processor.from_pretrained.assert_not_called()
def test_applies_pil_backend_only_to_image_processor(self):
config = SimpleNamespace(model_type="test_vlm", auto_map={})
for processor_kwargs in (
{"image_processor_backend": "pil"},
{"use_fast": False},
):
with self.subTest(processor_kwargs=processor_kwargs):
loaded_processor = MagicMock()
loaded_processor.image_processor.backend = "torchvision"
loaded_processor.tokenizer.chat_template = "template"
pil_processor = MagicMock(backend="pil")
auto_config = MagicMock()
auto_config.from_pretrained.return_value = config
auto_processor = MagicMock()
auto_processor.from_pretrained.return_value = loaded_processor
auto_image_processor = MagicMock()
auto_image_processor.from_pretrained.return_value = pil_processor
with patch.multiple(
processor_utils,
AutoConfig=auto_config,
AutoProcessor=auto_processor,
AutoImageProcessor=auto_image_processor,
):
processor = processor_utils.get_processor(
"test-model", **processor_kwargs
)
call_kwargs = auto_processor.from_pretrained.call_args.kwargs
self.assertNotIn("backend", call_kwargs)
self.assertNotIn("use_fast", call_kwargs)
auto_image_processor.from_pretrained.assert_called_once_with(
"test-model",
trust_remote_code=False,
revision=None,
backend="pil",
)
self.assertIs(processor.image_processor, pil_processor)
def test_resolves_model_name_before_loading_config(self):
remote_model = "s3://bucket/model"
local_model = "/cache/model"
test/registered/vlm/test_token_id_retokenize_e2e.py
+1 -3
View File
@@ -53,9 +53,7 @@ def _build_drift_prompt(model, image_token):
token), followed by one image placeholder. drift_delta is how many extra
tokens the non-canonical form carries vs. the canonical re-tokenization.
"""
tok = AutoProcessor.from_pretrained(
model, trust_remote_code=True, use_fast=True
).tokenizer
tok = AutoProcessor.from_pretrained(model, trust_remote_code=True).tokenizer
def enc(text):
return tok.encode(text, add_special_tokens=False)
test/registered/vlm/test_vision_openai_server_extra.py
+1 -1
View File
@@ -14,7 +14,7 @@ class TestQwen3OmniServer(OmniOpenAITestMixin):
extra_args = [ # workaround to fit into H100
"--mem-fraction-static=0.90",
"--disable-cuda-graph",
"--disable-fast-image-processor",
"--image-processor-backend=pil",
"--grammar-backend=none",
]
test/registered/vlm/test_vlm_input_format.py
+1 -1
View File
@@ -74,7 +74,7 @@ class VLMInputTestBase:
cls.main_image.append(Image.open(BytesIO(response.content)))
cls.processor = AutoProcessor.from_pretrained(
cls.model_path, trust_remote_code=True, use_fast=True
cls.model_path, trust_remote_code=True
)
_fix_added_tokens_encoding(cls.processor.tokenizer)
cls._init_visual()