From 25c9f724d4785eaa6921690d1634d20a3faac57b Mon Sep 17 00:00:00 2001 From: "Zhuangcheng(Jesse) Gu" <40918450+Chokoyo@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:20:21 -0400 Subject: [PATCH] fix(multimodal): handle tensor images in exact-token preprocessing (#30368) Signed-off-by: Zhuangcheng(Jesse) Gu Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Mick --- .../sglang/srt/multimodal/processors/base_processor.py | 9 ++++++++- test/registered/vlm/test_token_id_retokenize_e2e.py | 7 +++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 9c6d172b0..eadbc77fa 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -1732,7 +1732,14 @@ class BaseMultimodalProcessor(ABC): """ assert images is not None - image_sizes = [(image.height, image.width) for image in images] + image_sizes = [ + ( + tuple(image.shape[-2:]) + if isinstance(image, torch.Tensor) + else (image.height, image.width) + ) + for image in images + ] num_image_tokens = self._processor._get_num_multimodal_tokens( image_sizes=image_sizes ).num_image_tokens diff --git a/test/registered/vlm/test_token_id_retokenize_e2e.py b/test/registered/vlm/test_token_id_retokenize_e2e.py index af37df453..419d09a7a 100644 --- a/test/registered/vlm/test_token_id_retokenize_e2e.py +++ b/test/registered/vlm/test_token_id_retokenize_e2e.py @@ -9,6 +9,9 @@ With SGLANG_MM_AVOID_RETOKENIZE ON (default), the server keeps the user's original tokens verbatim and only expands the image placeholder, so prompt_tokens stays faithful to what the client sent. +The test uses JPEG so CUDA decoding returns a CHW tensor, covering the same +exact-token path as PIL-backed images. + For each model we launch a real server twice with the same predefined, non-canonical prompt ("Describe" split into "D"+"escribe") plus one image: @@ -42,8 +45,8 @@ register_cpu_ci(est_time=123, suite="stage-b-test-cpu-intel") def _data_uri(): img = Image.new("RGB", (64, 64), (128, 128, 128)) buf = io.BytesIO() - img.save(buf, format="PNG") - return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + img.save(buf, format="JPEG") + return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() def _build_drift_prompt(model, image_token):